Merge pull request #4 from sitewards/AdHoc_add_syslog

Add Syslog as logging src and refector old code
sitewards · Oct 28, 2019 · 535f565 · 535f565
2 parents f6dbaca + db84ac5
commit 535f565
Show file tree

Hide file tree

Showing 4 changed files with 145 additions and 48 deletions.
diff --git a/README.md b/README.md
@@ -48,7 +48,19 @@ it is also possible to parse log file manually and feed it in the script:
 grep nginx /var/log/syslog | grep -v "health_check" | tail -1  | php monitor.php --interval 20minutes --termination "/sbin/shutdown -h now"
 ```
 
-PLEASE NOTE: only the last line of STDIN will be processed in this case, `--logfile` is ignored in this mode.
+PLEASE NOTE: Only the last line of STDIN will be processed. STDIN will only be processed, if no other input is given by `--logfile` or `--syslog`
+
+### Parsing Syslog
+For phrasing the syslog and check for log entries from a specific unit, you can use the following:
+```
+php monitor.php --termination "/sbin/shutdown -h now" --syslog "nginx.service" --interval "4hours"
+```
+or in your ansible config
+```
+sitewards_server_suicide_logfile_path: ""
+sitewards_server_suicide_syslog_unit: "nginx.service"
+```
+
 
 ### Parsing log file with external tool, using pipe and terminating the self with AWS
 

diff --git a/defaults/main.yml b/defaults/main.yml
@@ -4,6 +4,7 @@ sitewards_server_suicide_group: "root"
 
 sitewards_server_suicide_cronjob_execution_interval: "*/15"
 sitewards_server_suicide_logfile_path: "/var/log/nginx/access.log"
+sitewards_server_suicide_syslog_unit: ""
 sitewards_server_suicide_server_termination_check_interval: "4hours"
 sitewards_server_suicide_base_path: "/opt/sitewards/server-suicide"
 sitewards_server_suicide_server_id: ""

diff --git a/tasks/main.yml b/tasks/main.yml
@@ -27,6 +27,6 @@
     name: monitor server termination
     user: "{{ sitewards_server_suicide_owner }}"
     minute: "{{ sitewards_server_suicide_cronjob_execution_interval }}"
-    job: "{{ sitewards_server_suicide_stdin_data_feed }} | php {{ sitewards_server_suicide_base_path }}/monitor.php --termination \"{{ sitewards_server_suicide_termination_url }}\" --logfile \"{{ sitewards_server_suicide_logfile_path }}\" --interval \"{{ sitewards_server_suicide_server_termination_check_interval }}\""
+    job: "{{ sitewards_server_suicide_stdin_data_feed }} | php {{ sitewards_server_suicide_base_path }}/monitor.php --termination \"{{ sitewards_server_suicide_termination_url }}\" --logfile \"{{ sitewards_server_suicide_logfile_path }}\" --syslog \"{{ sitewards_server_suicide_syslog_unit }}\" --interval \"{{ sitewards_server_suicide_server_termination_check_interval }}\""
     state: present
     disabled: "{{ not sitewards_server_suicide_active }}"
diff --git a/templates/monitor.php b/templates/monitor.php
@@ -5,37 +5,48 @@
  * @copyright   Copyright (c) Sitewards GmbH (https://www.sitewards.com/)
  */
 
+/**
+ * @return int
+ * @throws Exception
+ */
 function main()
 {
-    $stdinBuffer = get_stdin();
-    $hasStdin    = strlen($stdinBuffer) > 0;
-
-    $options = getopt('', ['logfile:', 'termination:', 'interval:']);
-    validate_options($options, !$hasStdin);
-
-    $current_date_time = new DateTimeImmutable();
+    $options = getopt('', ['logfile:', 'termination:', 'interval:', 'syslog:']);
+    validate_options($options);
+    $expiryThreshold = new DateTimeImmutable('-' . $options['interval']);
 
-    if (!is_uptime_expired($options['interval'])) {
+    if (!is_uptime_expired($expiryThreshold)) {
         return 0;
     }
 
-    if ($hasStdin) {
-        if (is_stdin_expired($stdinBuffer, $options['interval'])) {
-            trigger_termination($options['termination']);
-
-            return 1;
-        }
+    $terminate = false;
+    if (!empty($options['logfile'])) {
+        $terminate = is_access_log_expired($options['logfile'], $expiryThreshold);
+    } elseif (!empty($options['syslog'])){
+        $terminate = is_sys_log_expired($options['syslog'], $expiryThreshold);
     } else {
-        if (is_access_log_expired($options['logfile'], $options['interval'])) {
-            trigger_termination($options['termination']);
-
-            return 1;
+        $stdinBuffer = get_stdin();
+        // check if standard input has content
+        if(strlen($stdinBuffer) > 0){
+            $terminate = is_stdin_expired($stdinBuffer, $expiryThreshold);
+        } else {
+            throw new RuntimeException('No input from standard in. Access log file or system log must be specified.');
         }
     }
 
+    if($terminate){
+        trigger_termination($options['termination']);
+        return 1;
+    }
+
     return 0;
 }
 
+/**
+ * Get the input from standard input and convert it into a string.
+ *
+ * @return string
+ */
 function get_stdin()
 {
     $stdinHandle = fopen('php://stdin', 'r');
@@ -59,64 +70,121 @@ function get_stdin()
     return $stdinBufferPrev;
 }
 
-
-function is_stdin_expired($stdin, $logExpiryInterval)
+/**
+ * Checks the specific text for log entries. Get the last line and compare it to the given threshold.
+ *
+ * @param                   $stdin
+ * @param DateTimeImmutable $expiryThreshold
+ *
+ * @return bool
+ */
+function is_stdin_expired($stdin, DateTimeImmutable $expiryThreshold): bool
 {
-    // get datetime from the line and compare it to the current datetime
-    // to identify date in log regexp searches for a first value between square brackets that have at least 10 chars
-    preg_match('/\[([^\]]{10,}?)\]/', $stdin, $matches);
+    $lastLogEntry = get_date_from_text($stdin);
 
     // the access log is empty, or wrong format, consider it expired
-    if (empty($matches)) {
-        return 1;
+    if (!$lastLogEntry) {
+        return true;
     }
 
-    $lastLogEntry    = new DateTimeImmutable($matches[1]);
-    $expiryThreshold = new DateTimeImmutable('-' . $logExpiryInterval);
-
     // Check if latest access log entry is higher than the defined time interval
     return $lastLogEntry < $expiryThreshold;
 }
 
-
-function is_access_log_expired($logfile, $logExpiryInterval)
+/**
+ * Checks the specific file for log entries. Get the last line and compare it to the given threshold.
+ *
+ * @param                   $logfile
+ * @param DateTimeImmutable $expiryThreshold
+ *
+ * @return bool
+ */
+function is_access_log_expired($logfile, DateTimeImmutable $expiryThreshold): bool
 {
     // check last access time
     $logfileRef = escapeshellarg($logfile);
     $line       = `tail -n 1 $logfileRef`;
 
     // get datetime from the line and compare it to the current datetime
-    preg_match('/\[([^\[\]]*:[^\[\]]*)\]/', $line, $matches);
+    $lastLogEntry = get_date_from_text($line);
 
     // the access log is empty, or wrong format, let's try to check mtime
-    if (empty($matches)) {
+    if (!$lastLogEntry) {
         $lastLogEntry = DateTimeImmutable::createFromFormat('U', filemtime($logfile));
-    } else {
-        $lastLogEntry = new DateTimeImmutable($matches[1]);
     }
 
-    $expiryThreshold = new DateTimeImmutable('-' . $logExpiryInterval);
-
     // Check if latest access log entry is higher than the defined time interval
     return $lastLogEntry < $expiryThreshold;
 }
 
-function is_uptime_expired($uptimeExpiryInterval)
+/**
+ * Tries to find a date in a given text and convert it to a date.
+ * In case it cannot find any date or it cannot be converted, this function will return false.
+ *
+ * @param $text
+ *
+ * @return bool|DateTimeImmutable
+ */
+function get_date_from_text($text){
+    // to identify date in log regexp searches for a first value between square brackets that have at least 10 chars
+    preg_match('/\[([^\]]{10,}?)\]/', $text, $matches);
+    if(empty($matches)){
+        return false;
+    } else {
+        try {
+            return new DateTimeImmutable($matches[1]);
+        } catch (Exception $e){
+            return false;
+        }
+
+    }
+}
+
+/**
+ * Checks for the last log entry in the specified unit and compare it to the expiry threshold.
+ *
+ * @param                   $unit
+ * @param DateTimeImmutable $expiryThreshold
+ *
+ * @return bool
+ */
+function is_sys_log_expired($unit, DateTimeImmutable $expiryThreshold): bool
+{
+    $unitRef      = escapeshellarg($unit);
+    $lastLogEvent = json_decode(`/bin/journalctl -u $unitRef -n 1 -o json`, true);
+    if (empty($lastLogEvent)) {
+        // the access log is empty, or wrong format, consider it expired
+        return false;
+    }
+    // convert milliseconds into seconds from begin of unix epoch
+    $lastLogTimeSeconds = substr($lastLogEvent['__REALTIME_TIMESTAMP'], 0, -6);
+    $lastLogEntry       = DateTimeImmutable::createFromFormat('U', $lastLogTimeSeconds);
+    return $lastLogEntry < $expiryThreshold;
+}
+
+/**
+ * Check if server is up for longer than the defined time interval
+ *
+ * @param $expiryThreshold
+ *
+ * @return bool
+ * @throws Exception
+ */
+function is_uptime_expired($expiryThreshold): bool
 {
     $upSince         = new DateTimeImmutable(shell_exec('uptime -s'));
-    $expiryThreshold = new DateTimeImmutable('-' . $uptimeExpiryInterval);
-
-    // Check if server is up for longer than the defined time interval
     return $upSince < $expiryThreshold;
 }
 
-function validate_options(&$options, $logfileRequired)
+/**
+ * Check if the command line options are valid and sets default values
+ *
+ * @param $options
+ */
+function validate_options(&$options)
 {
-    if ($logfileRequired && !isset($options['logfile'])) {
-        throw new RuntimeException('Access log file must be specified.');
-    }
 
-    if ($logfileRequired && !file_exists($options['logfile'])) {
+    if (!empty($options['logfile']) && !file_exists($options['logfile'])) {
         throw new RuntimeException('Access log file does not exist.');
     }
 
@@ -130,6 +198,12 @@ function validate_options(&$options, $logfileRequired)
     $options['interval'] = trim($options['interval']);
 }
 
+
+/**
+ * Commit Suicide
+ *
+ * @param $urlOrShellCommand
+ */
 function trigger_termination($urlOrShellCommand)
 {
     if (filter_var($urlOrShellCommand, FILTER_VALIDATE_URL) !== false) {
@@ -139,11 +213,21 @@ function trigger_termination($urlOrShellCommand)
     }
 }
 
+/**
+ * Run a shell command that terminates the instance
+ *
+ * @param $shellCommand
+ */
 function trigger_termination_via_cli($shellCommand)
 {
     system($shellCommand);
 }
 
+/**
+ * Sends request to an url and ask for termination
+ *
+ * @param $uri
+ */
 function trigger_termination_via_url($uri)
 {
     $ch = curl_init();
@@ -154,7 +238,7 @@ function trigger_termination_via_url($uri)
     curl_setopt($ch, CURLOPT_TIMEOUT, 300);
     curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 0);
 
-    $response = curl_exec($ch);
+    curl_exec($ch);
 
     curl_close($ch);
 }