fix tests and improve workflow

.github/workflows/deploy.yaml

@@ -0,0 +1,30 @@
+name: Build and Deploy to dockerhub
+on:
+  workflow_dispatch:
+  release:
+    types: [published]
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.9'
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: logzio/logzio-api-fetcher
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/tests.yaml

@@ -0,0 +1,33 @@
+name: Automatic tests and code-coverage
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.9'
+      - name: Add credentials to config files
+        run: |
+          cd tests
+          grep -rli '<<CISCO_SECURE_X_API_ID>>' * | xargs -i@ sed -i 's/<<CISCO_SECURE_X_API_ID>>/${{ secrets.CISCO_SECURE_X_API_ID }}/g' @
+          grep -rli '<<CISCO_SECURE_X_API_KEY>>' * | xargs -i@ sed -i 's/<<CISCO_SECURE_X_API_KEY>>/${{ secrets.CISCO_SECURE_X_API_KEY }}/g' @
+          grep -rli '<<AZURE_AD_SECRET_ID>>' * | xargs -i@ sed -i 's/<<AZURE_AD_SECRET_ID>>/${{ secrets.AZURE_AD_SECRET_ID }}/g' @
+          grep -rli '<<AZURE_AD_SECRET_VALUE>>' * | xargs -i@ sed -i 's/<<AZURE_AD_SECRET_VALUE>>/${{ secrets.AZURE_AD_SECRET_VALUE }}/g' @
+          grep -rli '<<CLIENT_ID>>' * | xargs -i@ sed -i 's/<<CLIENT_ID>>/${{ secrets.AZURE_AD_CLIENT_ID }}/g' @
+          grep -rli '<<AZURE_AD_TENANT_ID>>' * | xargs -i@ sed -i 's/<<AZURE_AD_TENANT_ID>>/${{ secrets.AZURE_AD_TENANT_ID }}/g' @
+      - name: Install dependencies
+        run: |
+          pip install -r requirements.txt
+          pip install pytest-cov
+          pip install pytest
+      - name: Run unit tests
+        run: |
+          pytest -s --log-cli-level=debug tests/*_tests.py

requirements.txt

@@ -1,5 +1,5 @@
-pyyaml
-requests
-urllib3
-python-dateutil
-jsonpath-ng
+pyyaml~=6.0.1
+requests~=2.31.0
+python-dateutil~=2.9.0.post0
+jsonpath-ng
+responses~=0.25.0

src/apis_manager.py

@@ -33,12 +33,13 @@ class ApisManager:
     AUTH_API_TYPES = [API_GENERAL_TYPE, API_CISCO_SECURE_X_TYPE]
     OAUTH_API_TYPES = [API_GENERAL_TYPE, API_AZURE_GRAPH_TYPE, API_AZURE_MAIL_REPORTS_TYPE]
 
-    def __init__(self) -> None:
+    def __init__(self, test=False) -> None:
         self._apis: list[Api] = []
         self._logzio_connection: Optional[LogzioConnection] = None
         self._threads = []
         self._event = threading.Event()
         self._lock = threading.Lock()
+        self.test = test
 
     def run(self) -> None:
         if not self._read_data_from_config():
@@ -96,6 +97,7 @@ def _add_oauth_api(self, oauth_api_data: OAuthApiData) -> None:
             self._apis.append(AzureMailReports(oauth_api_data))
 
     def _run_api_scheduled_task(self, api: Api) -> None:
+        logger.info("NAAMA TEST 6")
         logzio_shipper = LogzioShipper(self._logzio_connection.url, self._logzio_connection.token)
 
         for api_custom_field in api.get_api_custom_fields():
@@ -107,8 +109,11 @@ def _run_api_scheduled_task(self, api: Api) -> None:
             thread.start()
             thread.join()
 
-            if self._event.wait(timeout=api.get_api_time_interval()):
+            if self._event.wait(timeout=api.get_api_time_interval()) or self.test:
+                logger.info("NAAMA TEST 7")
                 break
+            else:
+                logger.info("NAAMA TEST NOT 7")
 
     def _send_data_to_logzio(self, api: Api, logzio_shipper: LogzioShipper) -> None:
         logger.info("Task is running for api {}...".format(api.get_api_name()))
@@ -152,6 +157,10 @@ def _send_data_to_logzio(self, api: Api, logzio_shipper: LogzioShipper) -> None:
             "Task is over. A new Task for api {0} will run in {1} minute/s.".format(api.get_api_name(),
                                                                                     int(api.get_api_time_interval() / 60)))
 
+        if self.test:
+            logger.info("NAAMA TEST 10")
+            os.kill(os.getpid(), signal.SIGTERM)
+
     def _write_last_start_date_to_file(self, api_name: str, last_start_date: str) -> None:
         self._lock.acquire()
 

src/cisco_secure_x.py

@@ -16,6 +16,7 @@ class CiscoSecureX(AuthApi):
     HTTP_METHOD = 'GET'
     URL = 'https://api.amp.cisco.com/v1/events'
     START_DATE_NAME = 'start_date'
+    END_DATE_NAME = 'end_date'
     NEXT_URL_JSON_PATH = 'metadata.links.next'
     DATA_JSON_PATH = 'data'
     DATA_DATE_JSON_PATH = 'date'
@@ -25,6 +26,7 @@ def __init__(self, api_base_data: AuthApiBaseData) -> None:
         json_paths = ApiJsonPaths(api_next_url_json_path=CiscoSecureX.NEXT_URL_JSON_PATH,
                                   api_data_json_path=CiscoSecureX.DATA_JSON_PATH,
                                   api_data_date_json_path=CiscoSecureX.DATA_DATE_JSON_PATH)
-        general_type_data = ApiGeneralTypeData(CiscoSecureX.START_DATE_NAME, json_paths)
+        general_type_data = ApiGeneralTypeData(CiscoSecureX.START_DATE_NAME, CiscoSecureX.END_DATE_NAME, json_paths)
+
         super().__init__(api_base_data,
                          AuthApiGeneralTypeData(general_type_data, http_request))

src/oauth_api.py

@@ -131,4 +131,3 @@ def get_token_request(self):
     def _set_current_data_last_date(self, date):
         if date:
             self._current_data_last_date = date
-

tests/azure_graph_api_tests.py

@@ -3,8 +3,7 @@
 import multiprocessing
 import json
 import math
-
-import httpretty
+import responses
 
 from src.azure_graph import AzureGraph
 from src.logzio_shipper import LogzioShipper
@@ -82,17 +81,20 @@ def test_fetch_data_with_filters(self) -> None:
         self.assertNotEqual(total_data_num, fetched_data_num)
 
     def test_sending_data(self) -> None:
+        logger.info("NAAMA TEST 1")
         queue = multiprocessing.Queue()
         self.tests_utils.start_process_and_wait_until_finished(queue,
                                                                AzureGraphApiTests.BASE_CONFIG_FILE,
                                                                self.tests_utils.run_oauth_api_process,
                                                                status=200,
                                                                sleep_time=10)
 
-        requests_num, sent_logs_num, sent_bytes = queue.get()
+        logger.info("NAAMA TEST 2")
+        requests_num, sent_logs_num, sent_bytes = queue.get(False)
         data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data(
             self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK])
 
+        logger.info("NAAMA TEST 3")
         self.assertEqual(math.ceil(sent_bytes / LogzioShipper.MAX_BULK_SIZE_BYTES), requests_num)
         self.assertEqual(data_num, sent_logs_num)
         self.assertEqual(data_bytes, sent_bytes)
@@ -105,7 +107,7 @@ def test_sending_data_iterations(self) -> None:
                                                                status=200,
                                                                sleep_time=70)
 
-        requests_num, sent_logs_num, sent_bytes = queue.get()
+        requests_num, sent_logs_num, sent_bytes = queue.get(False)
         data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data(
             self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK])
 
@@ -121,7 +123,7 @@ def test_sending_data_multiple_azure_graph(self) -> None:
                                                                status=200,
                                                                sleep_time=10)
 
-        requests_num, sent_logs_num, sent_bytes = queue.get()
+        requests_num, sent_logs_num, sent_bytes = queue.get(False)
         data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data(
             self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK])
 
@@ -137,7 +139,7 @@ def test_sending_data_with_custom_fields(self) -> None:
                                                                status=200,
                                                                sleep_time=10)
 
-        requests_num, sent_logs_num, sent_bytes = queue.get()
+        requests_num, sent_logs_num, sent_bytes = queue.get(False)
         data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data(
             self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK])
         custom_fields_azure_graph = self.tests_utils.get_first_api(AzureGraphApiTests.CUSTOM_FIELDS_CONFIG_FILE,
@@ -156,21 +158,21 @@ def test_time_interval(self) -> None:
                                                                status=200,
                                                                sleep_time=70)
 
-        requests_num, sent_logs_num, sent_bytes = queue.get()
+        requests_num, sent_logs_num, sent_bytes = queue.get(False)
         data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data(
             self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK])
 
         self.assertEqual(math.ceil(sent_bytes / LogzioShipper.MAX_BULK_SIZE_BYTES), requests_num)
         self.assertEqual(data_num, sent_logs_num)
         self.assertEqual(data_bytes, sent_bytes)
 
-    @httpretty.activate
+    @responses.activate
     def test_last_start_date(self) -> None:
-        httpretty.register_uri(httpretty.GET, self.AZURE_GRAPH_TEST_URL,
+        responses.add(responses.GET, self.AZURE_GRAPH_TEST_URL,
                                body=json.dumps(AzureGraphApiTests.azure_graph_json_body), status=200)
         azure_graph = self.tests_utils.get_first_api(AzureGraphApiTests.CUSTOM_FIELDS_CONFIG_FILE,
                                                      is_auth_api=False)
-        httpretty.register_uri(httpretty.POST, azure_graph.get_token_request.url,
+        responses.add(responses.POST, azure_graph.get_token_request.url,
                                body=json.dumps(AzureGraphApiTests.azure_graph_token_json_body), status=200)
 
         for _ in azure_graph.fetch_data():
@@ -188,7 +190,7 @@ def test_bad_config(self) -> None:
                                                                status=200,
                                                                sleep_time=1)
 
-        requests_num, sent_logs_num, sent_bytes = queue.get()
+        requests_num, sent_logs_num, sent_bytes = queue.get(False)
 
         self.assertEqual(0, requests_num)
         self.assertEqual(0, sent_logs_num)

tests/cisco_secure_x_api_tests.py

@@ -3,8 +3,7 @@
 import multiprocessing
 import json
 import math
-
-import httpretty
+import responses
 
 from src.cisco_secure_x import CiscoSecureX
 from src.logzio_shipper import LogzioShipper
@@ -83,7 +82,7 @@ def test_sending_data(self) -> None:
                                                                status=200,
                                                                sleep_time=10)
 
-        requests_num, sent_logs_num, sent_bytes = queue.get()
+        requests_num, sent_logs_num, sent_bytes = queue.get(False)
         data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data(
             self.cisco_secure_x_json_body['data'])
 
@@ -99,7 +98,7 @@ def test_sending_data_iterations(self) -> None:
                                                                status=200,
                                                                sleep_time=70)
 
-        requests_num, sent_logs_num, sent_bytes = queue.get()
+        requests_num, sent_logs_num, sent_bytes = queue.get(False)
         data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data(
             self.cisco_secure_x_json_body['data'])
 
@@ -115,7 +114,7 @@ def test_sending_data_multiple_cisco_secure_x(self) -> None:
                                                                status=200,
                                                                sleep_time=10)
 
-        requests_num, sent_logs_num, sent_bytes = queue.get()
+        requests_num, sent_logs_num, sent_bytes = queue.get(False)
         data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data(
             self.cisco_secure_x_json_body['data'])
 
@@ -131,7 +130,7 @@ def test_sending_data_with_custom_fields(self) -> None:
                                                                status=200,
                                                                sleep_time=10)
 
-        requests_num, sent_logs_num, sent_bytes = queue.get()
+        requests_num, sent_logs_num, sent_bytes = queue.get(False)
         data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data(
             self.cisco_secure_x_json_body['data'])
         custom_fields_cisco_secure_x = self.tests_utils.get_first_api(CiscoSecureXApiTests.CUSTOM_FIELDS_CONFIG_FILE,
@@ -150,17 +149,17 @@ def test_time_interval(self) -> None:
                                                                status=200,
                                                                sleep_time=70)
 
-        requests_num, sent_logs_num, sent_bytes = queue.get()
+        requests_num, sent_logs_num, sent_bytes = queue.get(False)
         data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data(
             self.cisco_secure_x_json_body['data'])
 
         self.assertEqual(math.ceil(sent_bytes / LogzioShipper.MAX_BULK_SIZE_BYTES), requests_num)
         self.assertEqual(data_num, sent_logs_num)
         self.assertEqual(data_bytes, sent_bytes)
 
-    @httpretty.activate
+    @responses.activate
     def test_last_start_date(self) -> None:
-        httpretty.register_uri(httpretty.GET, CiscoSecureX.URL,
+        responses.add(responses.GET, CiscoSecureX.URL,
                                body=json.dumps(CiscoSecureXApiTests.cisco_secure_x_json_body), status=200)
 
         base_cisco_secure_x = self.tests_utils.get_first_api(CiscoSecureXApiTests.CUSTOM_FIELDS_CONFIG_FILE,
@@ -181,7 +180,7 @@ def test_bad_config(self) -> None:
                                                                status=200,
                                                                sleep_time=1)
 
-        requests_num, sent_logs_num, sent_bytes = queue.get()
+        requests_num, sent_logs_num, sent_bytes = queue.get(False)
 
         self.assertEqual(0, requests_num)
         self.assertEqual(0, sent_logs_num)

tests/combined_api_tests.py

@@ -114,7 +114,7 @@ def test_sending_data(self) -> None:
                                                                sleep_time=10,
                                                                is_multi_test=True)
 
-        requests_num, sent_logs_num, sent_bytes = queue.get()
+        requests_num, sent_logs_num, sent_bytes = queue.get(False)
         data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data(
             self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK])
 

tests/config/azure_graph/base_config.yaml

@@ -24,3 +24,4 @@ oauth_apis:
       time_interval: 1
       days_back_fetch: 30
     start_date_name: createdDateTime
+    end_date_name: EndDate