diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml new file mode 100644 index 0000000..ca2a9ed --- /dev/null +++ b/.github/workflows/deploy.yaml @@ -0,0 +1,30 @@ +name: Build and Deploy to dockerhub +on: + workflow_dispatch: + release: + types: [published] +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.9' + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: logzio/logzio-api-fetcher + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml new file mode 100644 index 0000000..cd74583 --- /dev/null +++ b/.github/workflows/tests.yaml @@ -0,0 +1,33 @@ +name: Automatic tests and code-coverage +on: + push: + branches: + - main + pull_request: + branches: + - main +jobs: + tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.9' + - name: Add credentials to config files + run: | + cd tests + grep -rli '<>' * | xargs -i@ sed -i 's/<>/${{ secrets.CISCO_SECURE_X_API_ID }}/g' @ + grep -rli '<>' * | xargs -i@ sed -i 's/<>/${{ secrets.CISCO_SECURE_X_API_KEY }}/g' @ + grep -rli '<>' * | xargs -i@ sed -i 's/<>/${{ secrets.AZURE_AD_SECRET_ID }}/g' @ + grep -rli '<>' * | xargs -i@ sed -i 's/<>/${{ secrets.AZURE_AD_SECRET_VALUE }}/g' @ + grep -rli '<>' * | xargs -i@ sed -i 's/<>/${{ secrets.AZURE_AD_CLIENT_ID }}/g' @ + grep -rli '<>' * | xargs -i@ sed -i 's/<>/${{ secrets.AZURE_AD_TENANT_ID }}/g' @ + - name: Install dependencies + run: | + pip install -r requirements.txt + pip install pytest-cov + pip install pytest + - name: Run unit tests + run: | + pytest -s --log-cli-level=debug tests/*_tests.py \ No newline at end of file diff --git a/.github/workflows/workflow.yaml b/.github/workflows/workflow.yaml deleted file mode 100644 index 965d42a..0000000 --- a/.github/workflows/workflow.yaml +++ /dev/null @@ -1,64 +0,0 @@ -name: Automatic tests, code-coverage and -on: - workflow_dispatch: - push: - branches: - - main -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: '3.x' - - name: Add credentials to config files - run: | - cd tests - grep -rli '<>' * | xargs -i@ sed -i 's/<>/${{ secrets.CISCO_SECURE_X_API_ID }}/g' @ - grep -rli '<>' * | xargs -i@ sed -i 's/<>/${{ secrets.CISCO_SECURE_X_API_KEY }}/g' @ - grep -rli '<>' * | xargs -i@ sed -i 's/<>/${{ secrets.AZURE_AD_SECRET_ID }}/g' @ - grep -rli '<>' * | xargs -i@ sed -i 's/<>/${{ secrets.AZURE_AD_SECRET_VALUE }}/g' @ - grep -rli '<>' * | xargs -i@ sed -i 's/<>/${{ secrets.AZURE_AD_CLIENT_ID }}/g' @ - grep -rli '<>' * | xargs -i@ sed -i 's/<>/${{ secrets.AZURE_AD_TENANT_ID }}/g' @ - - name: Run unit tests - run: | - pip install pytest - pip install httpretty - pip install requests - pip install pyyaml - pip install jsonpath-ng - pip install python-dateutil - pip install pytest-cov - pytest --cov-report xml:code_coverage.xml --cov=src tests/*_tests.py - - name: Code-coverage - run: | - # Get line-rate - line_rate=$(head -2 code_coverage.xml | tail -1 | egrep -o "line-rate=\"[0-1]\.?[0-9]*\"" | egrep -o "[0-1]\.?[0-9]*") - - # Print line-rate - echo | awk -v num=$line_rate '{ printf "line-rate: %d%\n", (num * 100) }' - - # Check code-coverage conditions - echo | awk -v num=$line_rate '{ if (num < 0.8) { printf "line-rate is less than 80%"; exit 1 } else { exit 0 }}' - exit_code=$? - if [ $exit_code -eq 1 ]; then - exit 1 - fi - - name: Log in to Docker Hub - uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 - with: - images: my-docker-hub-namespace/my-docker-hub-repository - - name: Build and push Docker image - uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc - with: - context: . - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} diff --git a/requirements.txt b/requirements.txt index 94b0ab9..e52a63a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -pyyaml -requests -urllib3 -python-dateutil -jsonpath-ng \ No newline at end of file +pyyaml~=6.0.1 +requests~=2.31.0 +python-dateutil~=2.9.0.post0 +jsonpath-ng +responses~=0.25.0 \ No newline at end of file diff --git a/src/apis_manager.py b/src/apis_manager.py index 05470ed..512f20e 100644 --- a/src/apis_manager.py +++ b/src/apis_manager.py @@ -33,12 +33,13 @@ class ApisManager: AUTH_API_TYPES = [API_GENERAL_TYPE, API_CISCO_SECURE_X_TYPE] OAUTH_API_TYPES = [API_GENERAL_TYPE, API_AZURE_GRAPH_TYPE, API_AZURE_MAIL_REPORTS_TYPE] - def __init__(self) -> None: + def __init__(self, test=False) -> None: self._apis: list[Api] = [] self._logzio_connection: Optional[LogzioConnection] = None self._threads = [] self._event = threading.Event() self._lock = threading.Lock() + self.test = test def run(self) -> None: if not self._read_data_from_config(): @@ -96,6 +97,7 @@ def _add_oauth_api(self, oauth_api_data: OAuthApiData) -> None: self._apis.append(AzureMailReports(oauth_api_data)) def _run_api_scheduled_task(self, api: Api) -> None: + logger.info("NAAMA TEST 6") logzio_shipper = LogzioShipper(self._logzio_connection.url, self._logzio_connection.token) for api_custom_field in api.get_api_custom_fields(): @@ -107,8 +109,11 @@ def _run_api_scheduled_task(self, api: Api) -> None: thread.start() thread.join() - if self._event.wait(timeout=api.get_api_time_interval()): + if self._event.wait(timeout=api.get_api_time_interval()) or self.test: + logger.info("NAAMA TEST 7") break + else: + logger.info("NAAMA TEST NOT 7") def _send_data_to_logzio(self, api: Api, logzio_shipper: LogzioShipper) -> None: logger.info("Task is running for api {}...".format(api.get_api_name())) @@ -152,6 +157,10 @@ def _send_data_to_logzio(self, api: Api, logzio_shipper: LogzioShipper) -> None: "Task is over. A new Task for api {0} will run in {1} minute/s.".format(api.get_api_name(), int(api.get_api_time_interval() / 60))) + if self.test: + logger.info("NAAMA TEST 10") + os.kill(os.getpid(), signal.SIGTERM) + def _write_last_start_date_to_file(self, api_name: str, last_start_date: str) -> None: self._lock.acquire() diff --git a/src/cisco_secure_x.py b/src/cisco_secure_x.py index ebc9845..9db3b08 100644 --- a/src/cisco_secure_x.py +++ b/src/cisco_secure_x.py @@ -16,6 +16,7 @@ class CiscoSecureX(AuthApi): HTTP_METHOD = 'GET' URL = 'https://api.amp.cisco.com/v1/events' START_DATE_NAME = 'start_date' + END_DATE_NAME = 'end_date' NEXT_URL_JSON_PATH = 'metadata.links.next' DATA_JSON_PATH = 'data' DATA_DATE_JSON_PATH = 'date' @@ -25,6 +26,7 @@ def __init__(self, api_base_data: AuthApiBaseData) -> None: json_paths = ApiJsonPaths(api_next_url_json_path=CiscoSecureX.NEXT_URL_JSON_PATH, api_data_json_path=CiscoSecureX.DATA_JSON_PATH, api_data_date_json_path=CiscoSecureX.DATA_DATE_JSON_PATH) - general_type_data = ApiGeneralTypeData(CiscoSecureX.START_DATE_NAME, json_paths) + general_type_data = ApiGeneralTypeData(CiscoSecureX.START_DATE_NAME, CiscoSecureX.END_DATE_NAME, json_paths) + super().__init__(api_base_data, AuthApiGeneralTypeData(general_type_data, http_request)) diff --git a/src/oauth_api.py b/src/oauth_api.py index 8e2ea83..21ff475 100644 --- a/src/oauth_api.py +++ b/src/oauth_api.py @@ -131,4 +131,3 @@ def get_token_request(self): def _set_current_data_last_date(self, date): if date: self._current_data_last_date = date - diff --git a/tests/azure_graph_api_tests.py b/tests/azure_graph_api_tests.py index 784f67f..1821e2e 100644 --- a/tests/azure_graph_api_tests.py +++ b/tests/azure_graph_api_tests.py @@ -3,8 +3,7 @@ import multiprocessing import json import math - -import httpretty +import responses from src.azure_graph import AzureGraph from src.logzio_shipper import LogzioShipper @@ -82,6 +81,7 @@ def test_fetch_data_with_filters(self) -> None: self.assertNotEqual(total_data_num, fetched_data_num) def test_sending_data(self) -> None: + logger.info("NAAMA TEST 1") queue = multiprocessing.Queue() self.tests_utils.start_process_and_wait_until_finished(queue, AzureGraphApiTests.BASE_CONFIG_FILE, @@ -89,10 +89,12 @@ def test_sending_data(self) -> None: status=200, sleep_time=10) - requests_num, sent_logs_num, sent_bytes = queue.get() + logger.info("NAAMA TEST 2") + requests_num, sent_logs_num, sent_bytes = queue.get(False) data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) + logger.info("NAAMA TEST 3") self.assertEqual(math.ceil(sent_bytes / LogzioShipper.MAX_BULK_SIZE_BYTES), requests_num) self.assertEqual(data_num, sent_logs_num) self.assertEqual(data_bytes, sent_bytes) @@ -105,7 +107,7 @@ def test_sending_data_iterations(self) -> None: status=200, sleep_time=70) - requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) @@ -121,7 +123,7 @@ def test_sending_data_multiple_azure_graph(self) -> None: status=200, sleep_time=10) - requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) @@ -137,7 +139,7 @@ def test_sending_data_with_custom_fields(self) -> None: status=200, sleep_time=10) - requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) custom_fields_azure_graph = self.tests_utils.get_first_api(AzureGraphApiTests.CUSTOM_FIELDS_CONFIG_FILE, @@ -156,7 +158,7 @@ def test_time_interval(self) -> None: status=200, sleep_time=70) - requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) @@ -164,13 +166,13 @@ def test_time_interval(self) -> None: self.assertEqual(data_num, sent_logs_num) self.assertEqual(data_bytes, sent_bytes) - @httpretty.activate + @responses.activate def test_last_start_date(self) -> None: - httpretty.register_uri(httpretty.GET, self.AZURE_GRAPH_TEST_URL, + responses.add(responses.GET, self.AZURE_GRAPH_TEST_URL, body=json.dumps(AzureGraphApiTests.azure_graph_json_body), status=200) azure_graph = self.tests_utils.get_first_api(AzureGraphApiTests.CUSTOM_FIELDS_CONFIG_FILE, is_auth_api=False) - httpretty.register_uri(httpretty.POST, azure_graph.get_token_request.url, + responses.add(responses.POST, azure_graph.get_token_request.url, body=json.dumps(AzureGraphApiTests.azure_graph_token_json_body), status=200) for _ in azure_graph.fetch_data(): @@ -188,7 +190,7 @@ def test_bad_config(self) -> None: status=200, sleep_time=1) - requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) self.assertEqual(0, requests_num) self.assertEqual(0, sent_logs_num) diff --git a/tests/cisco_secure_x_api_tests.py b/tests/cisco_secure_x_api_tests.py index 2310148..84ec857 100644 --- a/tests/cisco_secure_x_api_tests.py +++ b/tests/cisco_secure_x_api_tests.py @@ -3,8 +3,7 @@ import multiprocessing import json import math - -import httpretty +import responses from src.cisco_secure_x import CiscoSecureX from src.logzio_shipper import LogzioShipper @@ -83,7 +82,7 @@ def test_sending_data(self) -> None: status=200, sleep_time=10) - requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( self.cisco_secure_x_json_body['data']) @@ -99,7 +98,7 @@ def test_sending_data_iterations(self) -> None: status=200, sleep_time=70) - requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( self.cisco_secure_x_json_body['data']) @@ -115,7 +114,7 @@ def test_sending_data_multiple_cisco_secure_x(self) -> None: status=200, sleep_time=10) - requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( self.cisco_secure_x_json_body['data']) @@ -131,7 +130,7 @@ def test_sending_data_with_custom_fields(self) -> None: status=200, sleep_time=10) - requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( self.cisco_secure_x_json_body['data']) custom_fields_cisco_secure_x = self.tests_utils.get_first_api(CiscoSecureXApiTests.CUSTOM_FIELDS_CONFIG_FILE, @@ -150,7 +149,7 @@ def test_time_interval(self) -> None: status=200, sleep_time=70) - requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( self.cisco_secure_x_json_body['data']) @@ -158,9 +157,9 @@ def test_time_interval(self) -> None: self.assertEqual(data_num, sent_logs_num) self.assertEqual(data_bytes, sent_bytes) - @httpretty.activate + @responses.activate def test_last_start_date(self) -> None: - httpretty.register_uri(httpretty.GET, CiscoSecureX.URL, + responses.add(responses.GET, CiscoSecureX.URL, body=json.dumps(CiscoSecureXApiTests.cisco_secure_x_json_body), status=200) base_cisco_secure_x = self.tests_utils.get_first_api(CiscoSecureXApiTests.CUSTOM_FIELDS_CONFIG_FILE, @@ -181,7 +180,7 @@ def test_bad_config(self) -> None: status=200, sleep_time=1) - requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) self.assertEqual(0, requests_num) self.assertEqual(0, sent_logs_num) diff --git a/tests/combined_api_tests.py b/tests/combined_api_tests.py index 35ee3ea..3bae1e4 100644 --- a/tests/combined_api_tests.py +++ b/tests/combined_api_tests.py @@ -114,7 +114,7 @@ def test_sending_data(self) -> None: sleep_time=10, is_multi_test=True) - requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) diff --git a/tests/config/azure_graph/base_config.yaml b/tests/config/azure_graph/base_config.yaml index 3257506..9298a76 100644 --- a/tests/config/azure_graph/base_config.yaml +++ b/tests/config/azure_graph/base_config.yaml @@ -24,3 +24,4 @@ oauth_apis: time_interval: 1 days_back_fetch: 30 start_date_name: createdDateTime + end_date_name: EndDate diff --git a/tests/config/azure_graph/custom_fields_config.yaml b/tests/config/azure_graph/custom_fields_config.yaml index 2b56f57..ab5b0b2 100644 --- a/tests/config/azure_graph/custom_fields_config.yaml +++ b/tests/config/azure_graph/custom_fields_config.yaml @@ -20,6 +20,7 @@ oauth_apis: url: https://graph.microsoft.com/v1.0/auditLogs/signIns method: GET start_date_name: createdDateTime + end_date_name: EndDate json_paths: data_date: createdDateTime settings: diff --git a/tests/config/azure_graph/days_back_fetch_config.yaml b/tests/config/azure_graph/days_back_fetch_config.yaml index 0f2f243..6420cc9 100644 --- a/tests/config/azure_graph/days_back_fetch_config.yaml +++ b/tests/config/azure_graph/days_back_fetch_config.yaml @@ -20,6 +20,7 @@ oauth_apis: url: https://graph.microsoft.com/v1.0/auditLogs/signIns method: GET start_date_name: createdDateTime + end_date_name: EndDate json_paths: data_date: createdDateTime additional_filters: diff --git a/tests/config/azure_graph/filters_config.yaml b/tests/config/azure_graph/filters_config.yaml index ab0e731..182b0ea 100644 --- a/tests/config/azure_graph/filters_config.yaml +++ b/tests/config/azure_graph/filters_config.yaml @@ -20,6 +20,7 @@ oauth_apis: url: https://graph.microsoft.com/v1.0/auditLogs/signIns method: GET start_date_name: createdDateTime + end_date_name: EndDate json_paths: data_date: createdDateTime filters: diff --git a/tests/different_types_auth_api_tests.py b/tests/different_types_auth_api_tests.py index ac30448..742adf5 100644 --- a/tests/different_types_auth_api_tests.py +++ b/tests/different_types_auth_api_tests.py @@ -37,7 +37,7 @@ def test_sending_data(self) -> None: status=200, sleep_time=10) - requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( self.cisco_secure_x_json_body['data']) @@ -53,7 +53,7 @@ def test_sending_data_iterations(self) -> None: status=200, sleep_time=70) - requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( self.cisco_secure_x_json_body['data']) diff --git a/tests/general_tests.py b/tests/general_tests.py index 8980e11..0e9cc21 100644 --- a/tests/general_tests.py +++ b/tests/general_tests.py @@ -3,7 +3,7 @@ import multiprocessing import json import math -import httpretty +import responses import requests from requests.sessions import InvalidSchema @@ -53,7 +53,7 @@ def test_task_scheduler(self): status=200, sleep_time=70) - requests_num, _, sent_bytes = queue.get() + requests_num, _, sent_bytes = queue.get(False) self.assertEqual(math.ceil(sent_bytes / 2 / LogzioShipper.MAX_BULK_SIZE_BYTES) * 2, requests_num) @@ -118,7 +118,7 @@ def test_send_retry_status_500(self): status=500, sleep_time=10) - requests_num, _, _ = queue.get() + requests_num, _, _ = queue.get(False) self.assertEqual(LogzioShipper.MAX_RETRIES + 1, requests_num) @@ -130,7 +130,7 @@ def test_send_retry_status_502(self) -> None: status=502, sleep_time=10) - requests_num, _, _ = queue.get() + requests_num, _, _ = queue.get(False) self.assertEqual(LogzioShipper.MAX_RETRIES + 1, requests_num) @@ -142,7 +142,7 @@ def test_send_retry_status_503(self) -> None: status=503, sleep_time=10) - requests_num, _, _ = queue.get() + requests_num, _, _ = queue.get(False) self.assertEqual(LogzioShipper.MAX_RETRIES + 1, requests_num) @@ -154,13 +154,13 @@ def test_send_retry_status_504(self) -> None: status=504, sleep_time=10) - requests_num, _, _ = queue.get() + requests_num, _, _ = queue.get(False) self.assertEqual(LogzioShipper.MAX_RETRIES + 1, requests_num) - @httpretty.activate + @responses.activate def test_send_bad_format(self) -> None: - httpretty.register_uri(httpretty.POST, self.tests_utils.LOGZIO_URL, status=400) + responses.add(responses.POST, self.tests_utils.LOGZIO_URL, status=400) logzio_shipper = LogzioShipper(self.tests_utils.LOGZIO_URL, self.tests_utils.LOGZIO_TOKEN) @@ -175,9 +175,9 @@ def test_sending_bad_logzio_url(self) -> None: self.assertRaises(requests.ConnectionError, logzio_shipper.send_to_logzio) - @httpretty.activate + @responses.activate def test_sending_bad_logzio_token(self) -> None: - httpretty.register_uri(httpretty.POST, self.tests_utils.LOGZIO_URL, status=401) + responses.add(responses.POST, self.tests_utils.LOGZIO_URL, status=401) logzio_shipper = LogzioShipper(self.tests_utils.LOGZIO_URL, self.tests_utils.LOGZIO_TOKEN) diff --git a/tests/general_type_auth_api_tests.py b/tests/general_type_auth_api_tests.py index 7885a13..d230dc1 100644 --- a/tests/general_type_auth_api_tests.py +++ b/tests/general_type_auth_api_tests.py @@ -3,7 +3,7 @@ import multiprocessing import json import math -import httpretty +import responses from src.cisco_secure_x import CiscoSecureX from src.logzio_shipper import LogzioShipper @@ -82,7 +82,7 @@ def test_sending_data(self) -> None: status=200, sleep_time=10) - requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( self.cisco_secure_x_json_body['data']) @@ -98,7 +98,7 @@ def test_sending_data_with_http_request_headers(self) -> None: status=200, sleep_time=10) - requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( self.cisco_secure_x_json_body['data']) @@ -114,7 +114,7 @@ def test_sending_data_iterations(self) -> None: status=200, sleep_time=70) - requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( self.cisco_secure_x_json_body['data']) @@ -130,7 +130,7 @@ def test_sending_data_multiple_general_type_apis(self) -> None: status=200, sleep_time=10) - requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( self.cisco_secure_x_json_body['data']) @@ -146,7 +146,7 @@ def test_sending_data_with_custom_fields(self) -> None: status=200, sleep_time=10) - requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( self.cisco_secure_x_json_body['data']) custom_fields_cisco_secure_x = self.tests_utils.get_first_api(GeneralTypeAuthApiTests.CUSTOM_FIELDS_CONFIG_FILE, @@ -165,7 +165,7 @@ def test_time_interval(self) -> None: status=200, sleep_time=70) - requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( self.cisco_secure_x_json_body['data']) @@ -173,9 +173,9 @@ def test_time_interval(self) -> None: self.assertEqual(data_num, sent_logs_num) self.assertEqual(data_bytes, sent_bytes) - @httpretty.activate + @responses.activate def test_last_start_date(self) -> None: - httpretty.register_uri(httpretty.GET, CiscoSecureX.URL, + responses.add(responses.GET, CiscoSecureX.URL, body=json.dumps(GeneralTypeAuthApiTests.cisco_secure_x_json_body), status=200) base_cisco_secure_x = self.tests_utils.get_first_api(GeneralTypeAuthApiTests.CUSTOM_FIELDS_CONFIG_FILE, @@ -196,7 +196,7 @@ def test_bad_config(self) -> None: status=200, sleep_time=1) - requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) self.assertEqual(0, requests_num) self.assertEqual(0, sent_logs_num) diff --git a/tests/general_type_oauth_api_tests.py b/tests/general_type_oauth_api_tests.py index 19a25b8..8d45d09 100644 --- a/tests/general_type_oauth_api_tests.py +++ b/tests/general_type_oauth_api_tests.py @@ -4,8 +4,7 @@ import json import math from urllib.parse import unquote - -import httpretty +import responses from src.azure_graph import AzureGraph from src.logzio_shipper import LogzioShipper @@ -88,7 +87,7 @@ def test_sending_data(self) -> None: status=200, sleep_time=10) - requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) @@ -104,7 +103,7 @@ def test_sending_data_with_http_request_headers(self) -> None: status=200, sleep_time=10) - requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) @@ -120,7 +119,7 @@ def test_sending_data_iterations(self) -> None: status=200, sleep_time=70) - requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) @@ -136,7 +135,7 @@ def test_sending_data_multiple_general_type_apis(self) -> None: status=200, sleep_time=10) - requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) @@ -152,7 +151,7 @@ def test_sending_data_with_custom_fields(self) -> None: status=200, sleep_time=10) - requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) custom_fields_azure_graph = self.tests_utils.get_first_api(GeneralTypeOAuthApiTests.CUSTOM_FIELDS_CONFIG_FILE, @@ -171,7 +170,7 @@ def test_time_interval(self) -> None: status=200, sleep_time=70) - requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) @@ -179,13 +178,13 @@ def test_time_interval(self) -> None: self.assertEqual(data_num, sent_logs_num) self.assertEqual(data_bytes, sent_bytes) - @httpretty.activate + @responses.activate def test_last_start_date(self) -> None: - httpretty.register_uri(httpretty.GET, self.AZURE_GRAPH_TEST_URL, + responses.add(responses.GET, self.AZURE_GRAPH_TEST_URL, body=json.dumps(GeneralTypeOAuthApiTests.azure_graph_json_body), status=200) base_azure_graph = self.tests_utils.get_first_api(GeneralTypeOAuthApiTests.CUSTOM_FIELDS_CONFIG_FILE, is_auth_api=False) - httpretty.register_uri(httpretty.POST, base_azure_graph.get_token_request.url, + responses.add(responses.POST, base_azure_graph.get_token_request.url, body=json.dumps(self.azure_graph_token_json_body), status=200) for _ in base_azure_graph.fetch_data(): @@ -203,7 +202,7 @@ def test_bad_config(self) -> None: status=200, sleep_time=1) - requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) self.assertEqual(0, requests_num) self.assertEqual(0, sent_logs_num) diff --git a/tests/tests_utils.py b/tests/tests_utils.py index 3ff75ec..db785f9 100644 --- a/tests/tests_utils.py +++ b/tests/tests_utils.py @@ -1,5 +1,5 @@ import logging -import httpretty +import responses import multiprocessing import requests import json @@ -23,10 +23,12 @@ class TestUtils: - LOGZIO_HTTPPRETTY_URL = 'https://listener.logz.io:8071/?token=123456789a&type=api_fetcher' + LOGZIO_MOCK_URL = ('https://listener.logz.io:8071/?token=123456789a&type=api_fetcher', + 'https://listener.logz.io:8071/?token=123456789a') LOGZIO_URL = 'https://listener.logz.io:8071' LOGZIO_TOKEN = '123456789a' LAST_START_DATES_FILE = 'tests/last_start_dates.txt' + TYPE_CHAR_COUNT = 23 # sending to logz adds ', "type": "api_fetcher"' to the request which is extra 23 chars def __init__(self, api_http_method: str, api_url: str, api_body: dict, token_http_method: str = None, token_url: str = None, token_body: dict = None, second_http_method: str = None, @@ -41,66 +43,69 @@ def __init__(self, api_http_method: str, api_url: str, api_body: dict, token_htt self.second_api_url = second_api_url self.second_api_body = second_api_body - def start_process_and_wait_until_finished(self, queue: multiprocessing.Queue, config_file: str, + @staticmethod + def start_process_and_wait_until_finished(queue: multiprocessing.Queue, config_file: str, delegate: Callable[[str], None], status: int, sleep_time: int, is_multi_test: bool = False) -> None: process = multiprocessing.Process(target=delegate, args=(config_file, status, queue, is_multi_test)) + logger.info("Naama test 3.5") process.start() + logger.info("Naama test 3.8") time.sleep(sleep_time) os.kill(process.pid, signal.SIGTERM) process.join() - @httpretty.activate + @responses.activate def run_auth_api_process(self, config_file: str, status: int, queue: multiprocessing.Queue, is_multi_test: bool) -> None: - httpretty.register_uri(self.api_http_method, self.api_url, body=json.dumps(self.api_body), status=200) - httpretty.register_uri(httpretty.POST, TestUtils.LOGZIO_URL, status=status) + responses.add(self.api_http_method, self.api_url, body=json.dumps(self.api_body), status=200) + responses.add(responses.POST, TestUtils.LOGZIO_URL, status=status) ApisManager.CONFIG_FILE = config_file ApisManager.LAST_START_DATES_FILE = TestUtils.LAST_START_DATES_FILE logzio_requests = [] - ApisManager().run() + ApisManager(True).run() - for request in httpretty.latest_requests(): - if request.url.startswith(self.api_url): + logger.info("TEST: filling the queue auth api process test") + for call in responses.calls: + if call.request.url.startswith(self.api_url): continue - logzio_requests.append(request) + logzio_requests.append(call.request) queue.put(self._get_sending_data_results(logzio_requests)) - @httpretty.activate + @responses.activate def run_oauth_api_process(self, config_file: str, status: int, queue: multiprocessing.Queue, is_multi_test: bool) -> None: + logger.info("NAAMA TEST 4.0") from tests.azure_graph_api_tests import AzureGraphApiTests - httpretty.register_uri(httpretty.POST, TestUtils.LOGZIO_URL, status=status) - httpretty.register_uri(self.token_http_method, - self.token_url, - body=json.dumps(self.token_body)) - httpretty.register_uri(self.api_http_method, self.api_url, body=json.dumps(self.api_body), status=200, - headers={AzureGraph.OAUTH_AUTHORIZATION_HEADER: - AzureGraphApiTests.AZURE_GRAPH_TEST_TOKEN}) + responses.add(responses.POST, TestUtils.LOGZIO_URL, status=status) + responses.add(self.token_http_method, self.token_url, body=json.dumps(self.token_body)) + responses.add(self.api_http_method, self.api_url, body=json.dumps(self.api_body), status=200, + headers={AzureGraph.OAUTH_AUTHORIZATION_HEADER: AzureGraphApiTests.AZURE_GRAPH_TEST_TOKEN}) if is_multi_test: - httpretty.register_uri(self.second_http_method, self.second_api_url, body=json.dumps(self.second_api_body), - status=200) + responses.add(self.second_http_method, self.second_api_url, body=json.dumps(self.second_api_body), + status=200) ApisManager.CONFIG_FILE = config_file ApisManager.LAST_START_DATES_FILE = TestUtils.LAST_START_DATES_FILE logzio_requests = [] + logger.info("NAAMA TEST 4") + ApisManager(True).run() - ApisManager().run() - - for request in httpretty.latest_requests(): - if request.url.startswith(self.api_url): + logger.info("NAAMA TEST 5") + for call in responses.calls: + if call.request.url.startswith(self.api_url): continue - logzio_requests.append(request) - + logzio_requests.append(call.request) queue.put(self._get_sending_data_results(logzio_requests)) - def get_first_api(self, config_file: str, is_auth_api: bool) -> Api: + @staticmethod + def get_first_api(config_file: str, is_auth_api: bool) -> Api: base_config_reader = ConfigReader(config_file, ApisManager.API_GENERAL_TYPE, ApisManager.AUTH_API_TYPES, ApisManager.OAUTH_API_TYPES) @@ -132,7 +137,6 @@ def get_cisco_secure_x_api_total_data_bytes_and_num(self, config_file: str) -> t response = requests.get(url=url, auth=(cisco_secure_x.base_data.credentials.id, cisco_secure_x.base_data.credentials.key)) json_data = json.loads(response.content) - data_bytes, data_num = self.get_api_data_bytes_and_num_from_json_data(json_data['data']) total_data_bytes += data_bytes total_data_num += data_num @@ -149,9 +153,9 @@ def get_cisco_secure_x_api_total_data_bytes_and_num(self, config_file: str) -> t def get_azure_graph_api_total_data_bytes_and_num(self, config_file: str) -> tuple[int, int]: config_reader = ConfigReader(config_file, ApisManager.API_GENERAL_TYPE, ApisManager.AUTH_API_TYPES, ApisManager.OAUTH_API_TYPES) - azure_graph = None total_data_bytes = 0 total_data_num = 0 + token = "" for oauth_api_data in config_reader.get_oauth_apis_data(): azure_graph = AzureGraph(oauth_api_data) @@ -159,9 +163,9 @@ def get_azure_graph_api_total_data_bytes_and_num(self, config_file: str) -> tupl url = self.api_url while True: - response = requests.get(url=url, headers={OAuthApi.OAUTH_AUTHORIZATION_HEADER: - token, - AzureGraph.OAUTH_TOKEN_REQUEST_CONTENT_TYPE: AzureGraph.OAUTH_APPLICATION_JSON_CONTENT_TYPE}) + response = requests.get(url=url, headers={OAuthApi.OAUTH_AUTHORIZATION_HEADER: token, + AzureGraph.OAUTH_TOKEN_REQUEST_CONTENT_TYPE: + AzureGraph.OAUTH_APPLICATION_JSON_CONTENT_TYPE}) json_data = json.loads(response.content) data_bytes, data_num = self.get_api_data_bytes_and_num_from_json_data( json_data[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) @@ -176,7 +180,8 @@ def get_azure_graph_api_total_data_bytes_and_num(self, config_file: str) -> tupl return total_data_bytes, total_data_num - def get_api_custom_fields_bytes(self, api: Api) -> int: + @staticmethod + def get_api_custom_fields_bytes(api: Api) -> int: custom_fields: dict = {} for custom_field in api.base_data.custom_fields: @@ -184,13 +189,15 @@ def get_api_custom_fields_bytes(self, api: Api) -> int: return len(json.dumps(custom_fields)) - def get_last_start_dates_file_lines(self) -> list[str]: + @staticmethod + def get_last_start_dates_file_lines() -> list[str]: with open(TestUtils.LAST_START_DATES_FILE, 'r') as file: file_lines = file.readlines() return file_lines - def get_api_fetch_data_bytes_and_num(self, api: Api) -> tuple[int, int]: + @staticmethod + def get_api_fetch_data_bytes_and_num(api: Api) -> tuple[int, int]: fetched_data_bytes = 0 fetched_data_num = 0 for data in api.fetch_data(): @@ -199,7 +206,8 @@ def get_api_fetch_data_bytes_and_num(self, api: Api) -> tuple[int, int]: return fetched_data_bytes, fetched_data_num - def get_api_data_bytes_and_num_from_json_data(self, json_data: list) -> tuple[int, int]: + @staticmethod + def get_api_data_bytes_and_num_from_json_data(json_data: list) -> tuple[int, int]: data_num = 0 data_bytes = 0 for data in json_data: @@ -214,16 +222,16 @@ def _get_sending_data_results(self, latest_requests: list) -> tuple[int, int, in sent_bytes = 0 for request in latest_requests: - if request.url == self.LOGZIO_HTTPPRETTY_URL: + if request.url in self.LOGZIO_MOCK_URL: requests_num += 1 try: - decompressed_gzip = gzip.decompress(request.parsed_body).splitlines() + decompressed_gzip = gzip.decompress(request.body).splitlines() except TypeError: continue for log in decompressed_gzip: sent_logs_num += 1 - sent_bytes += len(log) + sent_bytes += len(log) - self.TYPE_CHAR_COUNT - return int(requests_num / 2), int(sent_logs_num / 2), int(sent_bytes / 2) + return int(requests_num), int(sent_logs_num), int(sent_bytes)