Add 429 handler to task | update sdk | add unit test

plugins/mimecast/.CHECKSUM

@@ -1,7 +1,7 @@
 {
-	"spec": "35db88201d0c437148d3a5851253f0e4",
-	"manifest": "8be151ad8e7d232ce62fded2340d6841",
-	"setup": "e585cdea9a7cf94ca339985c0f2bbc7d",
+	"spec": "acba7c10cf5c9787cf3f11c76b1edc4d",
+	"manifest": "9f9a8a2c45095798625a37f3878b8dd4",
+	"setup": "5ba5d7927984907d1551077e7dde11ac",
 	"schemas": [
 		{
 			"identifier": "add_group_member/schema.py",

plugins/mimecast/Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=linux/amd64 rapid7/insightconnect-python-3-slim-plugin:6.1.2
+FROM --platform=linux/amd64 rapid7/insightconnect-python-3-slim-plugin:6.1.4
 
 LABEL organization=rapid7
 LABEL sdk=python

plugins/mimecast/bin/komand_mimecast

@@ -6,7 +6,7 @@ from sys import argv
 
 Name = "Mimecast"
 Vendor = "rapid7"
-Version = "5.3.18"
+Version = "5.3.19"
 Description = "[Mimecast](https://www.mimecast.com) is a set of cloud services designed to provide next generation protection against advanced email-borne threats such as malicious URLs, malware, impersonation attacks, as well as internally generated threats, with a focus on email security. This plugin utilizes the [Mimecast API](https://www.mimecast.com/developer/documentation)"
 
 

plugins/mimecast/help.md

@@ -1014,6 +1014,7 @@ For the Create Managed URL action, the URL must include `http://` or `https://`
 
 # Version History
 
+* 5.3.19 - Update Task `monitor_siem_logs` to delay retry if a rate limit error is returned from Mimecast
 * 5.3.18 - Fix task connection test | Trim whitespace from connection inputs | bump SDK to version 6.1.2
 * 5.3.17 - Task `monitor_siem_logs` update the mapping used for the USBCOM region
 * 5.3.16 - Task `monitor_siem_logs` Limit the number of events per run to 7500 | bump SDK to version 6.1.0

plugins/mimecast/komand_mimecast/tasks/monitor_siem_logs/task.py

@@ -10,9 +10,10 @@
 from insightconnect_plugin_runtime.helper import get_time_hours_ago
 
 from .schema import MonitorSiemLogsInput, MonitorSiemLogsOutput, MonitorSiemLogsState, Component
-from ...util.constants import IS_LAST_TOKEN_FIELD
-from ...util.event import EventLogs
-from ...util.exceptions import ApiClientException
+from komand_mimecast.util.constants import IS_LAST_TOKEN_FIELD
+from komand_mimecast.util.event import EventLogs
+from komand_mimecast.util.exceptions import ApiClientException
+from komand_mimecast.util.util import Utils
 
 FIRST_RUN_CUTOFF = 24
 NORMAL_RUNNING_CUTOFF = 24 * 7
@@ -23,6 +24,7 @@ class MonitorSiemLogs(insightconnect_plugin_runtime.Task):
     NEXT_TOKEN = "next_token"  # nosec
     HEADER_NEXT_TOKEN = "mc-siem-token"  # nosec
     STATUS_CODE = "status_code"  # nosec
+    RATE_LIMIT_DATETIME = "rate_limit_datetime"
 
     NORMAL_RUNNING_CUTOFF = "normal_running_cutoff"  # nosec
     LAST_LOG_LINE = "last_log_line"  # nosec
@@ -42,6 +44,7 @@ def run(  # pylint: disable=unused-argument # noqa: MC0001
         self, params={}, state={}, custom_config={}
     ) -> (List[Dict], Dict, Dict):
         try:
+            self.check_rate_limit(state)
             has_more_pages = False
             header_next_token = state.get(self.NEXT_TOKEN, "")
             normal_running_cutoff = state.get(self.NORMAL_RUNNING_CUTOFF, False)
@@ -81,6 +84,7 @@ def run(  # pylint: disable=unused-argument # noqa: MC0001
                         f"An exception has been raised during retrieval of siem logs. Status code: {error.status_code} "
                         f"Error: {error}, returning state={state}, has_more_pages={has_more_pages}"
                     )
+                    self.check_rate_limit_error(error.status_code, state)
                     return [], state, has_more_pages, error.status_code, error
 
                 # check if the hashed file list from the previous run is the same as this run
@@ -257,3 +261,25 @@ def _get_filter_time(self, custom_config: Dict[str, int], normal_running_cutoff:
         self.logger.info(f"The following filter time will be used: {filter_time}")
 
         return filter_time
+
+    def check_rate_limit(self, state: Dict):
+        rate_limited = state.get(self.RATE_LIMIT_DATETIME)
+        now = time()
+        if rate_limited:
+            rate_limit_string = Utils.convert_epoch_to_readable(rate_limited)
+            log_msg = f"Rate limit value stored in state: {rate_limit_string}. "
+            if rate_limited > now:
+                log_msg += "Still within rate limiting period, skipping task execution..."
+                self.logger.info(log_msg)
+                return [], state, False, 200, None
+
+            log_msg += "However no longer in rate limiting period, so task can be executed..."
+            del state[self.RATE_LIMIT_DATETIME]
+            self.logger.info(log_msg)
+
+    def check_rate_limit_error(self, status_code: int, state: dict) -> None:
+        if status_code == 429:
+            new_run_time = time() + 300
+            new_run_time_string = Utils.convert_epoch_to_readable(new_run_time)
+            self.logger.error(f"A rate limit error has occurred, task will resume after {new_run_time_string}")
+            state[self.RATE_LIMIT_DATETIME] = new_run_time

plugins/mimecast/komand_mimecast/util/api.py

@@ -168,11 +168,11 @@ def _is_last_token(self, request: requests.Response) -> bool:
         except json.JSONDecodeError:
             return False
 
-    def _check_rate_limiting(self, request):
+    def _check_rate_limiting(self, response):
         rate_limit_status_code = 429
-        if request.status_code == rate_limit_status_code:
+        if response.status_code == rate_limit_status_code:
             raise ApiClientException(
-                preset=PluginException.Preset.RATE_LIMIT, status_code=rate_limit_status_code, data=request.text
+                preset=PluginException.Preset.RATE_LIMIT, status_code=rate_limit_status_code, data=response.text
             )
 
     def _handle_status_code_response(self, response: requests.request, status_code: int):

plugins/mimecast/komand_mimecast/util/util.py

@@ -1,9 +1,14 @@
 from typing import Dict, Any, Union
+from datetime import datetime
 
 from komand_mimecast.util.constants import BASE_HOSTNAME_MAP, DEFAULT_REGION
 
 
 class Utils:
+    @staticmethod
+    def convert_epoch_to_readable(epoch_time: int) -> str:
+        return datetime.utcfromtimestamp(epoch_time).strftime("%Y-%m-%d %H:%M:%S")
+
     @staticmethod
     def prepare_base_url(region: str = DEFAULT_REGION) -> str:
         if region == "USBCOM":

plugins/mimecast/plugin.spec.yaml

@@ -17,15 +17,15 @@ links:
  - "[Mimecast](http://mimecast.com)"
 references:
  - "[Mimecast API](https://www.mimecast.com/developer/documentation)"
-version: 5.3.18
+version: 5.3.19
 connection_version: 5
 supported_versions: ["Mimecast API 2024-06-18"]
 vendor: rapid7
 support: rapid7
 cloud_ready: true
 sdk:
   type: slim
-  version: 6.1.2
+  version: 6.1.4
   user: nobody
 status: []
 resources:
@@ -40,6 +40,7 @@ hub_tags:
   keywords: [mimecast, email, cloud_enabled]
   features: []
 version_history:
+- "5.3.19 - Update Task `monitor_siem_logs` to delay retry if a rate limit error is returned from Mimecast"
 - "5.3.18 - Fix task connection test | Trim whitespace from connection inputs | bump SDK to version 6.1.2"
 - "5.3.17 - Task `monitor_siem_logs` update the mapping used for the USBCOM region"
 - "5.3.16 - Task `monitor_siem_logs` Limit the number of events per run to 7500 | bump SDK to version 6.1.0"

plugins/mimecast/setup.py

@@ -3,7 +3,7 @@
 
 
 setup(name="mimecast-rapid7-plugin",
-      version="5.3.18",
+      version="5.3.19",
       description="[Mimecast](https://www.mimecast.com) is a set of cloud services designed to provide next generation protection against advanced email-borne threats such as malicious URLs, malware, impersonation attacks, as well as internally generated threats, with a focus on email security. This plugin utilizes the [Mimecast API](https://www.mimecast.com/developer/documentation)",
       author="rapid7",
       author_email="",

plugins/mimecast/unit_test/test_monitor_siem_logs.py

@@ -64,6 +64,16 @@ def test_monitor_siem_logs_raises_401(self, _mock_data):
         self.assertEqual(new_state, state_params)  # we shouldn't change the state if we encounter an error
         self.assertEqual(type(error), ApiClientException)
 
+    def test_monitor_siem_logs_raises_429(self, _mock_data):
+        state_params = {"next_token": "force_429", "last_log_line": 0}
+        expected_state = state_params.copy()
+        expected_state.update({"rate_limit_datetime": 1641038700.0})
+        response, new_state, has_more_pages, status_code, error = self.task.run(params={}, state=state_params)
+        self.assertEqual(status_code, 429)
+        self.assertEqual(response, [])
+        self.assertEqual(new_state, expected_state)
+        self.assertEqual(type(error), ApiClientException)
+
     @patch("logging.Logger.error")
     def test_monitor_siem_logs_stops_path_traversal(self, mock_logger, _mock_data):
         test_state = {

plugins/mimecast/unit_test/util.py

@@ -76,11 +76,12 @@ def status_code(self):
                 return 200
 
         class MockResponseZip:
-            def __init__(self, status_code, content, headers, json):
+            def __init__(self, status_code, content, headers, json, text=""):
                 self.status_code = status_code
                 self.content = content
                 self.headers_value = headers
                 self.json_value = json
+                self.text = text
 
             @property
             def headers(self):
@@ -170,6 +171,22 @@ def json(self):
                     ],
                 }
                 resp = MockResponseZip(401, b"", {}, json.dumps(json_value))
+            if "force_429" in data:
+                json_value = {
+                    "meta": {"isLastToken": False, "status": 429},
+                    "fail": [
+                        {
+                            "errors": [
+                                {
+                                    "code": "err_xdk_invalid_signature",
+                                    "message": "0004 Invalid Signature",
+                                    "retryable": False,
+                                }
+                            ]
+                        }
+                    ],
+                }
+                resp = MockResponseZip(429, b"", {}, json.dumps(json_value))
             elif "force_json" in data:
                 resp = MockResponseZip(200, b'{ "type" : "MTA", "data" : ', headers, '{"meta": {"status": 200}}')
             elif "force_single_json_error" in data: