diff --git a/app/commons/clusterizer.py b/app/commons/clusterizer.py index 30bea243..67a80c15 100644 --- a/app/commons/clusterizer.py +++ b/app/commons/clusterizer.py @@ -23,7 +23,7 @@ from app.commons import logging from app.utils import utils, text_processing -logger = logging.getLogger("analyzerApp.clusterizer") +LOGGER = logging.getLogger("analyzerApp.clusterizer") class Clusterizer: @@ -65,7 +65,7 @@ def find_groups_by_similarity( rearranged_groups[cluster].append(real_id) new_group_id += 1 group_id = new_group_id - logger.debug("Time for finding groups: %.2f s", time() - start_time) + LOGGER.debug("Time for finding groups: %.2f s", time() - start_time) return rearranged_groups def similarity_groupping( @@ -125,7 +125,7 @@ def unite_groups_by_hashes(self, messages: list[str], threshold: float = 0.95) - if cluster not in rearranged_groups: rearranged_groups[cluster] = [] rearranged_groups[cluster].append(key) - logger.debug("Time for finding hash groups: %.2f s", time() - start_time) + LOGGER.debug("Time for finding hash groups: %.2f s", time() - start_time) return rearranged_groups def perform_light_deduplication(self, messages: list[str]) -> tuple[list[str], dict[int, list[int]]]: diff --git a/app/commons/esclient.py b/app/commons/esclient.py index 63258d5a..f7bbaf19 100644 --- a/app/commons/esclient.py +++ b/app/commons/esclient.py @@ -25,11 +25,9 @@ from urllib3.exceptions import InsecureRequestWarning from app.amqp.amqp import AmqpClient -from app.commons import logging +from app.commons import logging, request_factory, log_merger from app.commons.model.launch_objects import ApplicationConfig, Response, Launch, TestItem, BulkResponse from app.commons.model.ml import TrainInfo, ModelType -from app.commons.log_merger import LogMerger -from app.commons.log_requests import LogRequests from app.utils import utils, text_processing logger = logging.getLogger("analyzerApp.esclient") @@ -40,16 +38,12 @@ class EsClient: app_config: ApplicationConfig es_client: elasticsearch.Elasticsearch host: str - log_requests: LogRequests - log_merger: LogMerger tables_to_recreate: list[str] def __init__(self, app_config: ApplicationConfig, es_client: elasticsearch.Elasticsearch = None): self.app_config = app_config self.host = app_config.esHost self.es_client = es_client or self.create_es_client(app_config) - self.log_requests = LogRequests() - self.log_merger = LogMerger() self.tables_to_recreate = ["rp_aa_stats", "rp_model_train_stats", "rp_suggestions_info_metrics"] def create_es_client(self, app_config: ApplicationConfig) -> elasticsearch.Elasticsearch: @@ -220,7 +214,7 @@ def _to_index_bodies( if log.logLevel < utils.ERROR_LOGGING_LEVEL or not log.message.strip(): continue - bodies.append(LogRequests._prepare_log(launch, test_item, log, project_with_prefix)) + bodies.append(request_factory.prepare_log(launch, test_item, log, project_with_prefix)) logs_added = True if logs_added: test_item_ids.append(str(test_item.testItemId)) @@ -276,7 +270,7 @@ def _merge_logs(self, test_item_ids, project): test_items_dict[test_item_id] = [] test_items_dict[test_item_id].append(r) for test_item_id in test_items_dict: - merged_logs, _ = self.log_merger.decompose_logs_merged_and_without_duplicates( + merged_logs, _ = log_merger.decompose_logs_merged_and_without_duplicates( test_items_dict[test_item_id]) for log in merged_logs: if log["_source"]["is_merged"]: diff --git a/app/commons/log_merger.py b/app/commons/log_merger.py index 6274d442..ce832ecd 100644 --- a/app/commons/log_merger.py +++ b/app/commons/log_merger.py @@ -17,125 +17,141 @@ from app.utils import text_processing +FIELDS_TO_CLEAN = ["message", "detected_message", "detected_message_with_numbers", "detected_message_extended", + "message_extended", "message_without_params_extended", "message_without_params_and_brackets", + "detected_message_without_params_and_brackets"] +FIELDS_TO_MERGE = ["message", "found_exceptions", "potential_status_codes", "found_tests_and_methods", "only_numbers", + "urls", "paths", "message_params", "detected_message_without_params_extended", "whole_message"] + + +def _prepare_new_log(old_log: dict[str, Any], new_id, is_merged: bool, merged_small_logs: str, + fields_to_clean: Optional[list[str]] = None) -> dict[str, Any]: + """Prepare updated log""" + merged_log = copy.deepcopy(old_log) + merged_log["_source"]["is_merged"] = is_merged + merged_log["_id"] = new_id + merged_log["_source"]["merged_small_logs"] = merged_small_logs + if fields_to_clean: + for field in fields_to_clean: + merged_log["_source"][field] = "" + return merged_log + + +def merge_big_and_small_logs( + logs: list[dict[str, Any]], log_level_ids_to_add: dict[int, list[int]], + log_level_messages: dict[str, dict[int, str]], log_level_ids_merged: dict[int, dict[str, Any]], + logs_ids_in_merged_logs: dict[int, list[int]]) -> tuple[list[dict[str, Any]], dict[str, list[int]]]: + """Merge big message logs with small ones.""" + new_logs = [] + for log in logs: + if not log["_source"]["message"].strip(): + continue + log_level = log["_source"]["log_level"] + + if log["_id"] in log_level_ids_to_add[log_level]: + merged_small_logs = text_processing.compress(log_level_messages["message"][log_level]) + new_logs.append(_prepare_new_log(log, log["_id"], False, merged_small_logs)) + + log_ids_for_merged_logs = {} + for log_level in log_level_messages["message"]: + if not log_level_ids_to_add[log_level] and log_level_messages["message"][log_level].strip(): + log = log_level_ids_merged[log_level] + merged_logs_id = str(log["_id"]) + "_m" + new_log = _prepare_new_log( + log, merged_logs_id, True, text_processing.compress(log_level_messages["message"][log_level]), + fields_to_clean=FIELDS_TO_CLEAN) + log_ids_for_merged_logs[merged_logs_id] = logs_ids_in_merged_logs[log_level] + for field in log_level_messages: + if field == "message": + continue + if field == "whole_message": + new_log["_source"][field] = log_level_messages[field][log_level] + else: + new_log["_source"][field] = text_processing.compress( + log_level_messages[field][log_level]) + new_log["_source"]["found_exceptions_extended"] = text_processing.compress( + text_processing.enrich_found_exceptions(log_level_messages["found_exceptions"][log_level])) + + new_logs.append(new_log) + return new_logs, log_ids_for_merged_logs + + +def decompose_logs_merged_and_without_duplicates( + logs: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], dict[str, list[int]]]: + """Merge big logs with small ones without duplicates.""" + log_level_messages = {} + for field in FIELDS_TO_MERGE: + log_level_messages[field] = {} + log_level_ids_to_add = {} + log_level_ids_merged = {} + logs_unique_log_level = {} + logs_ids_in_merged_logs = {} + + for log in logs: + source = log['_source'] + if not source["message"].strip(): + continue + + log_level = source["log_level"] + + for field in log_level_messages: + if log_level not in log_level_messages[field]: + log_level_messages[field][log_level] = "" + if log_level not in log_level_ids_to_add: + log_level_ids_to_add[log_level] = [] + if log_level not in logs_unique_log_level: + logs_unique_log_level[log_level] = set() + + if source['message_lines'] <= 2 and source['message_words_number'] <= 100: + if log_level not in log_level_ids_merged: + log_level_ids_merged[log_level] = log + if log_level not in logs_ids_in_merged_logs: + logs_ids_in_merged_logs[log_level] = [] + logs_ids_in_merged_logs[log_level].append(log["_id"]) + + log_level_representative = log_level_ids_merged[log_level] + current_log_word_num = source["message_words_number"] + main_log_word_num = log_level_representative["_source"]["message_words_number"] + if current_log_word_num > main_log_word_num: + log_level_ids_merged[log_level] = log + + normalized_msg = " ".join(source["message"].strip().lower().split()) + if normalized_msg not in logs_unique_log_level[log_level]: + logs_unique_log_level[log_level].add(normalized_msg) -class LogMerger: - fields_to_clean: list[str] - fields_to_merge: list[str] - - def __init__(self): - self.fields_to_clean = ["message", "detected_message", - "detected_message_with_numbers", "stacktrace", - "detected_message_extended", - "stacktrace_extended", "message_extended", - "message_without_params_extended", - "message_without_params_and_brackets", - "detected_message_without_params_and_brackets"] - self.fields_to_merge = ["message", "found_exceptions", "potential_status_codes", - "found_tests_and_methods", "only_numbers", "urls", - "paths", "message_params", "detected_message_without_params_extended", - "whole_message"] - - def merge_big_and_small_logs( - self, logs: list[dict[str, Any]], log_level_ids_to_add: dict[int, list[int]], - log_level_messages: dict[str, dict[int, str]], log_level_ids_merged: dict[int, dict[str, Any]], - logs_ids_in_merged_logs: dict[int, list[int]]) -> tuple[list[dict[str, Any]], dict[str, list[int]]]: - """Merge big message logs with small ones.""" - new_logs = [] - for log in logs: - if not log["_source"]["message"].strip(): - continue - log_level = log["_source"]["log_level"] - - if log["_id"] in log_level_ids_to_add[log_level]: - merged_small_logs = text_processing.compress(log_level_messages["message"][log_level]) - new_logs.append(self.prepare_new_log(log, log["_id"], False, merged_small_logs)) - - log_ids_for_merged_logs = {} - for log_level in log_level_messages["message"]: - if not log_level_ids_to_add[log_level] and log_level_messages["message"][log_level].strip(): - log = log_level_ids_merged[log_level] - merged_logs_id = str(log["_id"]) + "_m" - new_log = self.prepare_new_log( - log, merged_logs_id, True, text_processing.compress(log_level_messages["message"][log_level]), - fields_to_clean=self.fields_to_clean) - log_ids_for_merged_logs[merged_logs_id] = logs_ids_in_merged_logs[log_level] for field in log_level_messages: - if field in ["message"]: - continue - if field in ["whole_message"]: - new_log["_source"][field] = log_level_messages[field][log_level] - else: - new_log["_source"][field] = text_processing.compress( - log_level_messages[field][log_level]) - new_log["_source"]["found_exceptions_extended"] = text_processing.compress( - text_processing.enrich_found_exceptions(log_level_messages["found_exceptions"][log_level])) - - new_logs.append(new_log) - return new_logs, log_ids_for_merged_logs - - def decompose_logs_merged_and_without_duplicates( - self, logs: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], dict[str, list[int]]]: - """Merge big logs with small ones without duplicates.""" - log_level_messages = {} - for field in self.fields_to_merge: - log_level_messages[field] = {} - log_level_ids_to_add = {} - log_level_ids_merged = {} - logs_unique_log_level = {} - logs_ids_in_merged_logs = {} - - for log in logs: - if not log["_source"]["message"].strip(): + if field in source: + splitter = "\n" if field in {"message", "whole_message"} else " " + log_level_messages[field][log_level] = \ + log_level_messages[field][log_level] + source[field] + splitter + + else: + log_level_ids_to_add[log_level].append(log["_id"]) + + return merge_big_and_small_logs( + logs, log_level_ids_to_add, log_level_messages, log_level_ids_merged, logs_ids_in_merged_logs) + + +def merge_logs( + logs: list[list[dict[str, Any]]], number_of_lines: int, + clean_numbers: bool) -> tuple[list[str], dict[int, dict[str, Any]], dict[str, list[int]]]: + full_log_ids_for_merged_logs = {} + log_messages = [] + log_dict = {} + ind = 0 + for prepared_log in logs: + merged_logs, log_ids_for_merged_logs = decompose_logs_merged_and_without_duplicates(prepared_log) + for _id, merged_list in log_ids_for_merged_logs.items(): + full_log_ids_for_merged_logs[_id] = merged_list + for log in merged_logs: + number_of_log_lines = number_of_lines + if log["_source"]["is_merged"]: + number_of_log_lines = -1 + log_message = text_processing.prepare_message_for_clustering( + log["_source"]["whole_message"], number_of_log_lines, clean_numbers) + if not log_message.strip(): continue - - log_level = log["_source"]["log_level"] - - for field in log_level_messages: - if log_level not in log_level_messages[field]: - log_level_messages[field][log_level] = "" - if log_level not in log_level_ids_to_add: - log_level_ids_to_add[log_level] = [] - if log_level not in logs_unique_log_level: - logs_unique_log_level[log_level] = set() - - if log["_source"]["original_message_lines"] <= 2 and \ - log["_source"]["original_message_words_number"] <= 100: - if log_level not in log_level_ids_merged: - log_level_ids_merged[log_level] = log - if log_level not in logs_ids_in_merged_logs: - logs_ids_in_merged_logs[log_level] = [] - logs_ids_in_merged_logs[log_level].append(log["_id"]) - - log_level_representative = log_level_ids_merged[log_level] - current_log_word_num = log["_source"]["original_message_words_number"] - main_log_word_num = log_level_representative["_source"]["original_message_words_number"] - if current_log_word_num > main_log_word_num: - log_level_ids_merged[log_level] = log - - normalized_msg = " ".join(log["_source"]["message"].strip().lower().split()) - if normalized_msg not in logs_unique_log_level[log_level]: - logs_unique_log_level[log_level].add(normalized_msg) - - for field in log_level_messages: - if field in log["_source"]: - splitter = "\n" if field in ["message", "whole_message"] else " " - log_level_messages[field][log_level] = \ - log_level_messages[field][log_level] + log["_source"][field] + splitter - - else: - log_level_ids_to_add[log_level].append(log["_id"]) - - return self.merge_big_and_small_logs( - logs, log_level_ids_to_add, log_level_messages, log_level_ids_merged, logs_ids_in_merged_logs) - - def prepare_new_log(self, old_log: dict[str, Any], new_id, is_merged: bool, merged_small_logs: str, - fields_to_clean: Optional[list[str]] = None) -> dict[str, Any]: - """Prepare updated log""" - merged_log = copy.deepcopy(old_log) - merged_log["_source"]["is_merged"] = is_merged - merged_log["_id"] = new_id - merged_log["_source"]["merged_small_logs"] = merged_small_logs - if fields_to_clean: - for field in fields_to_clean: - merged_log["_source"][field] = "" - return merged_log + log_messages.append(log_message) + log_dict[ind] = log + ind += 1 + return log_messages, log_dict, full_log_ids_for_merged_logs diff --git a/app/commons/log_requests.py b/app/commons/log_requests.py deleted file mode 100644 index 3744f388..00000000 --- a/app/commons/log_requests.py +++ /dev/null @@ -1,240 +0,0 @@ -# Copyright 2023 EPAM Systems -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from datetime import datetime -from typing import Any - -from app.commons.model.launch_objects import Launch, TestItem, Log, TestItemInfo -from app.commons.log_merger import LogMerger -from app.commons.prepared_log import PreparedLogMessage -from app.utils import utils, text_processing -from app.utils.log_preparation import basic_prepare - - -def create_log_template() -> dict: - return { - "_id": "", - "_index": "", - "_source": { - "launch_id": "", - "launch_name": "", - "launch_number": 0, - "launch_start_time": "", - "test_item": "", - "test_item_name": "", - "unique_id": "", - "cluster_id": "", - "cluster_message": "", - "test_case_hash": 0, - "is_auto_analyzed": False, - "issue_type": "", - "log_time": "", - "log_level": 0, - 'original_message': '', - "original_message_lines": 0, - "original_message_words_number": 0, - "message": "", - "is_merged": False, - "start_time": "", - "merged_small_logs": "", - "detected_message": "", - "detected_message_with_numbers": "", - "stacktrace": "", - "only_numbers": "", - "found_exceptions": "", - "whole_message": "", - "potential_status_codes": "", - "found_tests_and_methods": "", - "cluster_with_numbers": False - } - } - - -class LogRequests: - - def __init__(self): - self.log_merger = LogMerger() - - @staticmethod - def transform_issue_type_into_lowercase(issue_type): - return issue_type[:2].lower() + issue_type[2:] - - @staticmethod - def _fill_launch_test_item_fields(log_template: dict, launch: Launch, test_item: TestItem, project: str): - log_template["_index"] = project - log_template["_source"]["launch_id"] = launch.launchId - log_template["_source"]["launch_name"] = launch.launchName - log_template["_source"]["launch_number"] = getattr(launch, 'launchNumber', 0) - log_template["_source"]["launch_start_time"] = datetime( - *launch.launchStartTime[:6]).strftime("%Y-%m-%d %H:%M:%S") - log_template["_source"]["test_item"] = test_item.testItemId - log_template["_source"]["unique_id"] = test_item.uniqueId - log_template["_source"]["test_case_hash"] = test_item.testCaseHash - log_template["_source"]["is_auto_analyzed"] = test_item.isAutoAnalyzed - log_template["_source"]["test_item_name"] = text_processing.preprocess_test_item_name(test_item.testItemName) - log_template["_source"]["issue_type"] = LogRequests.transform_issue_type_into_lowercase(test_item.issueType) - log_template["_source"]["start_time"] = datetime(*test_item.startTime[:6]).strftime("%Y-%m-%d %H:%M:%S") - return log_template - - @staticmethod - def _fill_log_fields(log_template: dict, log: Log, number_of_lines: int) -> dict[str, Any]: - prepared_log = PreparedLogMessage(log.message, number_of_lines) - log_template["_id"] = log.logId - log_template["_source"]["log_time"] = datetime(*log.logTime[:6]).strftime("%Y-%m-%d %H:%M:%S") - log_template["_source"]["cluster_id"] = str(log.clusterId) - log_template["_source"]["cluster_message"] = log.clusterMessage - log_template["_source"]["cluster_with_numbers"] = utils.extract_clustering_setting(log.clusterId) - log_template["_source"]["log_level"] = log.logLevel - log_template["_source"]['original_message'] = log.message - log_template["_source"]["original_message_lines"] = text_processing.calculate_line_number( - prepared_log.clean_message) - log_template["_source"]["original_message_words_number"] = len( - text_processing.split_words(prepared_log.clean_message, split_urls=False)) - log_template["_source"]["message"] = prepared_log.message - log_template["_source"]["detected_message"] = prepared_log.exception_message_no_numbers - log_template["_source"]["detected_message_with_numbers"] = prepared_log.exception_message - log_template["_source"]["stacktrace"] = prepared_log.stacktrace - log_template["_source"]["only_numbers"] = prepared_log.exception_message_numbers - log_template["_source"]["urls"] = prepared_log.exception_message_urls - log_template["_source"]["paths"] = prepared_log.exception_message_paths - log_template["_source"]["message_params"] = prepared_log.exception_message_params - log_template["_source"]["found_exceptions"] = prepared_log.exception_found - log_template["_source"]["found_exceptions_extended"] = prepared_log.exception_found_extended - log_template["_source"]["detected_message_extended"] = \ - text_processing.enrich_text_with_method_and_classes(prepared_log.exception_message) - log_template["_source"]["detected_message_without_params_extended"] = \ - text_processing.enrich_text_with_method_and_classes(prepared_log.exception_message_no_params) - log_template["_source"]["stacktrace_extended"] = \ - text_processing.enrich_text_with_method_and_classes(prepared_log.stacktrace) - log_template["_source"]["message_extended"] = \ - text_processing.enrich_text_with_method_and_classes(prepared_log.message) - log_template["_source"]["message_without_params_extended"] = \ - text_processing.enrich_text_with_method_and_classes(prepared_log.message_no_params) - log_template["_source"]["whole_message"] = (prepared_log.exception_message_no_params + "\n" - + prepared_log.stacktrace) - log_template["_source"]["detected_message_without_params_and_brackets"] = \ - prepared_log.exception_message_no_params - log_template["_source"]["message_without_params_and_brackets"] = prepared_log.message_no_params - log_template["_source"]["potential_status_codes"] = prepared_log.exception_message_potential_status_codes - log_template["_source"]["found_tests_and_methods"] = prepared_log.test_and_methods_extended - - for field in ["message", "detected_message", "detected_message_with_numbers", - "stacktrace", "only_numbers", "found_exceptions", "found_exceptions_extended", - "detected_message_extended", "detected_message_without_params_extended", - "stacktrace_extended", "message_extended", "message_without_params_extended", - "detected_message_without_params_and_brackets", "message_without_params_and_brackets"]: - log_template["_source"][field] = text_processing.leave_only_unique_lines(log_template["_source"][field]) - log_template["_source"][field] = text_processing.clean_colon_stacking(log_template["_source"][field]) - return log_template - - @staticmethod - def _prepare_log(launch: Launch, test_item: TestItem, log: Log, project: str) -> dict: - log_template = create_log_template() - log_template = LogRequests._fill_launch_test_item_fields(log_template, launch, test_item, project) - log_template = LogRequests._fill_log_fields(log_template, log, launch.analyzerConfig.numberOfLogLines) - return log_template - - @staticmethod - def _fill_test_item_info_fields(log_template: dict, test_item_info: TestItemInfo, project: str) -> dict[str, Any]: - log_template["_index"] = project - log_template["_source"]["launch_id"] = test_item_info.launchId - log_template["_source"]["launch_name"] = test_item_info.launchName - log_template["_source"]["launch_number"] = getattr(test_item_info, 'launchNumber', 0) - log_template["_source"]["test_item"] = test_item_info.testItemId - log_template["_source"]["unique_id"] = test_item_info.uniqueId - log_template["_source"]["test_case_hash"] = test_item_info.testCaseHash - log_template["_source"]["test_item_name"] = text_processing.preprocess_test_item_name( - test_item_info.testItemName) - log_template["_source"]["is_auto_analyzed"] = False - log_template["_source"]["issue_type"] = "" - log_template["_source"]["start_time"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - return log_template - - @staticmethod - def _prepare_log_for_suggests(test_item_info: TestItemInfo, log: Log, project: str) -> dict: - log_template = create_log_template() - log_template = LogRequests._fill_test_item_info_fields(log_template, test_item_info, project) - log_template = LogRequests._fill_log_fields( - log_template, log, test_item_info.analyzerConfig.numberOfLogLines) - return log_template - - @staticmethod - def prepare_log_words(launches: list[Launch]) -> tuple[dict[str, int], int]: - log_words = {} - project = None - for launch in launches: - project = launch.project - for test_item in launch.testItems: - for log in test_item.logs: - if log.logLevel < utils.ERROR_LOGGING_LEVEL or not log.message.strip(): - continue - cleaned_message = basic_prepare(log.message) - det_message, stacktrace = text_processing.detect_log_description_and_stacktrace(cleaned_message) - for word in text_processing.split_words(stacktrace): - if '.' in word and len(word.split('.')) > 2: - log_words[word] = 1 - return log_words, project - - @staticmethod - def prepare_log_clustering_light(launch: Launch, test_item: TestItem, log: Log, project: str) -> dict[str, Any]: - log_template = create_log_template() - log_template = LogRequests._fill_launch_test_item_fields(log_template, launch, test_item, project) - prepared_log = PreparedLogMessage(log.message, -1) - log_template["_id"] = log.logId - log_template["_source"]["cluster_id"] = str(log.clusterId) - log_template["_source"]["cluster_message"] = log.clusterMessage - log_template["_source"]["log_level"] = log.logLevel - log_template["_source"]['original_message'] = log.message - log_template["_source"]["original_message_lines"] = text_processing.calculate_line_number( - prepared_log.clean_message) - log_template["_source"]["original_message_words_number"] = len( - text_processing.split_words(prepared_log.clean_message, split_urls=False)) - log_template["_source"]["message"] = text_processing.remove_numbers(prepared_log.message) - log_template["_source"]["detected_message"] = prepared_log.exception_message_no_numbers - log_template["_source"]["detected_message_with_numbers"] = prepared_log.exception_message - log_template["_source"]["stacktrace"] = prepared_log.stacktrace - log_template["_source"]["potential_status_codes"] = prepared_log.exception_message_potential_status_codes - log_template["_source"]["found_exceptions"] = prepared_log.exception_found - log_template["_source"]["whole_message"] = (prepared_log.exception_message_no_params + "\n" - + prepared_log.stacktrace) - return log_template - - def prepare_logs_for_clustering(self, launch: Launch, number_of_lines: int, clean_numbers: bool, - project: str) -> tuple[list[str], dict[int, dict[str, Any]], dict[str, list[int]]]: - log_messages = [] - log_dict = {} - ind = 0 - full_log_ids_for_merged_logs = {} - for test_item in launch.testItems: - prepared_logs = [] - for log in test_item.logs: - if log.logLevel < utils.ERROR_LOGGING_LEVEL: - continue - prepared_logs.append(LogRequests.prepare_log_clustering_light(launch, test_item, log, project)) - merged_logs, log_ids_for_merged_logs = self.log_merger.decompose_logs_merged_and_without_duplicates( - prepared_logs) - for _id, merged_list in log_ids_for_merged_logs.items(): - full_log_ids_for_merged_logs[_id] = merged_list - for log in merged_logs: - number_of_log_lines = number_of_lines - if log["_source"]["is_merged"]: - number_of_log_lines = -1 - log_message = text_processing.prepare_message_for_clustering( - log["_source"]["whole_message"], number_of_log_lines, clean_numbers) - if not log_message.strip(): - continue - log_messages.append(log_message) - log_dict[ind] = log - ind += 1 - return log_messages, log_dict, full_log_ids_for_merged_logs diff --git a/app/commons/model/ml.py b/app/commons/model/ml.py index 9effb5f5..34696caf 100644 --- a/app/commons/model/ml.py +++ b/app/commons/model/ml.py @@ -29,5 +29,5 @@ class ModelInfo(BaseModel): class TrainInfo(ModelInfo): - additional_projects: Optional[Iterable[int]] + additional_projects: Optional[Iterable[int]] = None gathered_metric_total: int = 0 diff --git a/app/commons/object_saving/filesystem_saver.py b/app/commons/object_saving/filesystem_saver.py index 2bf3b5a8..917912db 100644 --- a/app/commons/object_saving/filesystem_saver.py +++ b/app/commons/object_saving/filesystem_saver.py @@ -26,6 +26,10 @@ logger = logging.getLogger('analyzerApp.filesystemSaver') +def unify_path_separator(path: str) -> str: + return path.replace('\\', '/') + + class FilesystemSaver(Storage): _base_path: str @@ -34,48 +38,49 @@ def __init__(self, app_config: ApplicationConfig) -> None: def remove_project_objects(self, path: str, object_names: list[str]) -> None: for filename in object_names: - object_name_full = os.path.join(self._base_path, path, filename).replace("\\", "/") + object_name_full = unify_path_separator(os.path.join(self._base_path, path, filename)) if os.path.exists(object_name_full): os.remove(object_name_full) def put_project_object(self, data: Any, path: str, object_name: str, using_json: bool = False) -> None: - folder_to_save = os.path.join(self._base_path, path, os.path.dirname(object_name)).replace("\\", "/") - filename = os.path.join(self._base_path, path, object_name).replace("\\", "/") + folder_to_save = unify_path_separator(os.path.join(self._base_path, path, os.path.dirname(object_name))) + filename = unify_path_separator(os.path.join(self._base_path, path, object_name)) if folder_to_save: os.makedirs(folder_to_save, exist_ok=True) - with open(filename, "wb") as f: + with open(filename, 'wb') as f: if using_json: - f.write(json.dumps(data).encode("utf-8")) + f.write(json.dumps(data).encode('utf-8')) else: + # noinspection PyTypeChecker pickle.dump(data, f) logger.debug("Saved into folder '%s' with name '%s': %s", path, object_name, data) def get_project_object(self, path: str, object_name: str, using_json: bool = False) -> object | None: - filename = os.path.join(self._base_path, path, object_name).replace("\\", "/") + filename = unify_path_separator(os.path.join(self._base_path, path, object_name)) if not utils.validate_file(filename): raise ValueError(f'Unable to get file: {filename}') - with open(filename, "rb") as f: + with open(filename, 'rb') as f: return json.loads(f.read()) if using_json else pickle.load(f) def does_object_exists(self, path: str, object_name: str) -> bool: - return os.path.exists(os.path.join(self._base_path, path, object_name).replace("\\", "/")) + return os.path.exists(unify_path_separator(os.path.join(self._base_path, path, object_name))) def get_folder_objects(self, path: str, folder: str) -> list[str]: root_path = self._base_path if not root_path and not path: root_path = os.getcwd() - if folder.endswith('/'): - folder_to_check = os.path.join(root_path, path, folder).replace("\\", "/") + if unify_path_separator(folder).endswith('/'): + folder_to_check = unify_path_separator(os.path.join(root_path, path, folder)) if os.path.exists(folder_to_check): return [os.path.join(folder, file_name) for file_name in os.listdir(folder_to_check)] else: - folder_to_check = os.path.join(root_path, path).replace("\\", "/") + folder_to_check = unify_path_separator(os.path.join(root_path, path)) if os.path.exists(folder_to_check): return [file_name for file_name in os.listdir(folder_to_check) if file_name.startswith(folder)] return [] def remove_folder_objects(self, path: str, folder: str) -> bool: - folder_name = os.path.join(self._base_path, path, folder).replace("\\", "/") + folder_name = unify_path_separator(os.path.join(self._base_path, path, folder)) if os.path.exists(folder_name): shutil.rmtree(folder_name, ignore_errors=True) return True diff --git a/app/commons/object_saving/minio_client.py b/app/commons/object_saving/minio_client.py index 32b5b22f..18d31324 100644 --- a/app/commons/object_saving/minio_client.py +++ b/app/commons/object_saving/minio_client.py @@ -14,6 +14,7 @@ import io import json +import os.path import pickle from typing import Any @@ -30,12 +31,13 @@ class MinioClient(Storage): region: str bucket_prefix: str + minio_client: Minio def __init__(self, app_config: ApplicationConfig) -> None: minio_host = app_config.minioHost self.region = app_config.minioRegion self.bucket_prefix = app_config.minioBucketPrefix - self.minioClient = Minio( + self.minio_client = Minio( minio_host, access_key=app_config.minioAccessKey, secret_key=app_config.minioSecretKey, @@ -44,26 +46,44 @@ def __init__(self, app_config: ApplicationConfig) -> None: ) logger.info(f'Minio initialized {minio_host}') - def get_bucket(self, bucket: str | None): - if bucket: - return self.bucket_prefix + bucket - else: + def _get_project_name(self, project_id: str | None) -> str: + if not project_id: return '' + return self.bucket_prefix + project_id + + def get_bucket(self, bucket_id: str | None) -> str: + path = self._get_project_name(bucket_id) + if not path: + return path + + basename = os.path.basename(path) + if basename == path: + return path + return os.path.split(path)[0] + + def get_path(self, object_name: str, bucket_name: str, bucket_id: str | None) -> str: + path = self._get_project_name(bucket_id) + if not path or path == bucket_name: + return object_name + + path_octets = os.path.split(path)[1:] + return str(os.path.join(path_octets[0], *path_octets[1:], object_name)) def remove_project_objects(self, bucket: str, object_names: list[str]) -> None: bucket_name = self.get_bucket(bucket) - if not self.minioClient.bucket_exists(bucket_name): + if not self.minio_client.bucket_exists(bucket_name): return for object_name in object_names: - self.minioClient.remove_object(bucket_name=bucket_name, object_name=object_name) + path = self.get_path(object_name, bucket_name, bucket) + self.minio_client.remove_object(bucket_name=bucket_name, object_name=path) def put_project_object(self, data: Any, bucket: str, object_name: str, using_json=False) -> None: bucket_name = self.get_bucket(bucket) - if bucket_name: - if not self.minioClient.bucket_exists(bucket_name): - logger.debug("Creating minio bucket %s" % bucket_name) - self.minioClient.make_bucket(bucket_name=bucket_name, location=self.region) - logger.debug("Created minio bucket %s" % bucket_name) + path = self.get_path(object_name, bucket_name, bucket) + if bucket_name and not self.minio_client.bucket_exists(bucket_name): + logger.debug("Creating minio bucket %s" % bucket_name) + self.minio_client.make_bucket(bucket_name=bucket_name, location=self.region) + logger.debug("Created minio bucket %s" % bucket_name) if using_json: data_to_save = json.dumps(data).encode("utf-8") content_type = 'application/json' @@ -72,47 +92,55 @@ def put_project_object(self, data: Any, bucket: str, object_name: str, using_jso content_type = 'application/octet-stream' data_stream = io.BytesIO(data_to_save) data_stream.seek(0) - self.minioClient.put_object( - bucket_name=bucket_name, object_name=object_name, data=data_stream, length=len(data_to_save), + result = self.minio_client.put_object( + bucket_name=bucket_name, object_name=path, data=data_stream, length=len(data_to_save), content_type=content_type) - logger.debug("Saved into bucket '%s' with name '%s': %s", bucket_name, object_name, data) + etag = result[0] + logger.debug(f'Saved into bucket "{bucket_name}" with path "{path}", etag "{etag}": {data}') def get_project_object(self, bucket: str, object_name: str, using_json=False) -> object | None: bucket_name = self.get_bucket(bucket) + path = self.get_path(object_name, bucket_name, bucket) try: - obj = self.minioClient.get_object(bucket_name=bucket_name, object_name=object_name) + obj = self.minio_client.get_object(bucket_name=bucket_name, object_name=path) except NoSuchKey as exc: - raise ValueError(f'Unable to get file: {object_name}', exc) + raise ValueError(f'Unable to get file in bucket "{bucket_name}" with path "{path}"', exc) return json.loads(obj.data) if using_json else pickle.loads(obj.data) def does_object_exists(self, bucket: str, object_name: str) -> bool: bucket_name = self.get_bucket(bucket) - if bucket_name: - if not self.minioClient.bucket_exists(bucket_name): - return False + path = self.get_path(object_name, bucket_name, bucket) + if bucket_name and not self.minio_client.bucket_exists(bucket_name): + return False try: - self.minioClient.stat_object(bucket_name=bucket_name, object_name=object_name) + self.minio_client.stat_object(bucket_name=bucket_name, object_name=path) except NoSuchKey: return False return True def get_folder_objects(self, bucket: str, folder: str) -> list[str]: bucket_name = self.get_bucket(bucket) - if bucket_name: - if not self.minioClient.bucket_exists(bucket_name): - return [] + path = self.get_path(folder, bucket_name, bucket) + if bucket_name and not self.minio_client.bucket_exists(bucket_name): + return [] object_names = set() - object_list = self.minioClient.list_objects( - bucket_name, prefix=folder.endswith('/') and folder or folder + '/') + object_list = self.minio_client.list_objects(bucket_name, prefix=path.endswith('/') and path or path + '/') for obj in object_list: - object_names.add(obj.object_name.strip('/')) + object_name = obj.object_name.strip('/') + if folder != path: + # Bucket prefix includes path to the project + prefix = path[0: -(len(folder))] + object_name = object_name[len(prefix):] + object_names.add(object_name) return sorted(list(object_names)) def remove_folder_objects(self, bucket: str, folder: str) -> bool: bucket_name = self.get_bucket(bucket) - if bucket_name: - if not self.minioClient.bucket_exists(bucket_name): - return False - for obj in self.minioClient.list_objects(bucket_name, prefix=folder.endswith('/') and folder or folder + '/'): - self.minioClient.remove_object(bucket_name=bucket_name, object_name=obj.object_name) - return True + path = self.get_path(folder, bucket_name, bucket) + if bucket_name and not self.minio_client.bucket_exists(bucket_name): + return False + result = False + for obj in self.minio_client.list_objects(bucket_name, prefix=path.endswith('/') and path or path + '/'): + self.minio_client.remove_object(bucket_name=bucket_name, object_name=obj.object_name) + result = True + return result diff --git a/app/commons/prepared_log.py b/app/commons/prepared_log.py index 8d78f22b..a994aeb0 100644 --- a/app/commons/prepared_log.py +++ b/app/commons/prepared_log.py @@ -11,36 +11,40 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from typing import Optional -from app.utils.log_preparation import (basic_prepare, prepare_message, prepare_message_no_params, +from typing_extensions import override + +from app.utils import text_processing +from app.utils.log_preparation import (basic_prepare, clean_message, prepare_message, prepare_message_no_numbers, + prepare_message_no_params, prepare_exception_message_no_params_no_numbers, prepare_exception_message_no_params, prepare_exception_message_and_stacktrace) -from app.utils import text_processing class PreparedLogMessage: - original_message: str number_of_lines: int - _clean_message: str = None - _test_and_methods: set[str] = None - _message: str = None - _message_no_params: str = None - _exception_message: str = None - _stacktrace: str = None - _exception_message_urls: str = None - _exception_message_paths: str = None - _exception_message_potential_status_codes: str = None - _exception_message_params: str = None - _exception_message_no_params: str = None - _exception_message_no_numbers: str = None - _exception_message_numbers: str = None - _exception_found: str = None - _exception_found_extended: str = None - _test_and_methods_extended: str = None - _stacktrace_paths: str = None - _stacktrace_no_paths: str = None - _stacktrace_no_paths_extended: str = None + _basic_message: Optional[str] = None + _clean_message: Optional[str] = None + _test_and_methods: Optional[set[str]] = None + _message: Optional[str] = None + _message_no_params: Optional[str] = None + _exception_message: Optional[str] = None + _stacktrace: Optional[str] = None + _exception_message_urls: Optional[str] = None + _exception_message_paths: Optional[str] = None + _exception_message_potential_status_codes: Optional[str] = None + _exception_message_params: Optional[str] = None + _exception_message_no_params: Optional[str] = None + _exception_message_no_numbers: Optional[str] = None + _exception_message_numbers: Optional[str] = None + _exception_found: Optional[str] = None + _exception_found_extended: Optional[str] = None + _test_and_methods_extended: Optional[str] = None + _stacktrace_paths: Optional[str] = None + _stacktrace_no_paths: Optional[str] = None + _stacktrace_no_paths_extended: Optional[str] = None def __init__(self, message: str, number_of_lines: int): self.original_message = message @@ -49,10 +53,16 @@ def __init__(self, message: str, number_of_lines: int): def __str__(self): return self.original_message + @property + def basic_message(self) -> str: + if not self._basic_message: + self._basic_message = basic_prepare(self.original_message) + return self._basic_message + @property def clean_message(self) -> str: if not self._clean_message: - self._clean_message = basic_prepare(self.original_message) + self._clean_message = clean_message(self.basic_message) return self._clean_message @property @@ -64,7 +74,7 @@ def test_and_methods(self) -> set[str]: @property def message(self) -> str: if not self._message: - self._message = prepare_message(self.clean_message, self.number_of_lines, self.test_and_methods) + self._message = prepare_message_no_numbers(self.clean_message, self.number_of_lines, self.test_and_methods) return self._message @property @@ -82,8 +92,7 @@ def exception_message(self) -> str: @property def stacktrace(self) -> str: if not self._stacktrace: - self._raw_exception_message, self._stacktrace = prepare_exception_message_and_stacktrace( - self.clean_message) + self._exception_message, self._stacktrace = prepare_exception_message_and_stacktrace(self.clean_message) return self._stacktrace @property @@ -115,8 +124,8 @@ def exception_message_params(self) -> str: @property def exception_message_no_params(self) -> str: if not self._exception_message_no_params: - self._exception_message_no_params = text_processing.unify_spaces(prepare_exception_message_no_params( - self.exception_message)) + self._exception_message_no_params = prepare_exception_message_no_params_no_numbers( + self.exception_message) return self._exception_message_no_params @property @@ -171,3 +180,27 @@ def test_and_methods_extended(self) -> str: self._test_and_methods_extended = text_processing.enrich_text_with_method_and_classes( " ".join(self.test_and_methods)) return self._test_and_methods_extended + + +class PreparedLogMessageClustering(PreparedLogMessage): + + def __init__(self, message: str, number_of_lines: int) -> None: + super().__init__(message, number_of_lines) + + @override + @property + def clean_message(self) -> str: + return self.basic_message + + @override + @property + def message(self) -> str: + if not self._message: + self._message = prepare_message(self.clean_message, self.number_of_lines, self.test_and_methods) + return self._message + + @property + def exception_message_no_params(self) -> str: + if not self._exception_message_no_params: + self._exception_message_no_params = prepare_exception_message_no_params(self.exception_message) + return self._exception_message_no_params diff --git a/app/commons/request_factory.py b/app/commons/request_factory.py new file mode 100644 index 00000000..71d5838b --- /dev/null +++ b/app/commons/request_factory.py @@ -0,0 +1,214 @@ +# Copyright 2023 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This module contains functions for preparing requests to Vector DB such as Elasticsearch and OpenSearch.""" + +from datetime import datetime +from typing import Any + +from app.commons.model.launch_objects import Launch, TestItem, Log, TestItemInfo +from app.commons.prepared_log import PreparedLogMessage, PreparedLogMessageClustering +from app.utils import utils, text_processing +from app.utils.log_preparation import clean_message +from app.utils.utils import compute_if_absent + + +def create_log_template() -> dict: + return { + "_id": "", + "_index": "", + "_source": { + "launch_id": "", + "launch_name": "", + "launch_number": 0, + "launch_start_time": "", + "test_item": "", + "test_item_name": "", + "unique_id": "", + "cluster_id": "", + "cluster_message": "", + "test_case_hash": 0, + "is_auto_analyzed": False, + "issue_type": "", + "log_time": "", + "log_level": 0, + 'original_message': '', + 'message_lines': 0, + 'message_words_number': 0, + "message": "", + "is_merged": False, + "start_time": "", + "merged_small_logs": "", + "detected_message": "", + "detected_message_with_numbers": "", + "stacktrace": "", + "only_numbers": "", + "found_exceptions": "", + "whole_message": "", + "potential_status_codes": "", + "found_tests_and_methods": "", + "cluster_with_numbers": False + } + } + + +def transform_issue_type_into_lowercase(issue_type): + return issue_type[:2].lower() + issue_type[2:] + + +def _fill_launch_test_item_fields(log_template: dict, launch: Launch, test_item: TestItem, project: str): + log_template['_index'] = project + source = compute_if_absent(log_template, '_source', {}) + source["launch_id"] = launch.launchId + source["launch_name"] = launch.launchName + source["launch_number"] = getattr(launch, 'launchNumber', 0) + source["launch_start_time"] = datetime(*launch.launchStartTime[:6]).strftime("%Y-%m-%d %H:%M:%S") + source["test_item"] = test_item.testItemId + source["unique_id"] = test_item.uniqueId + source["test_case_hash"] = test_item.testCaseHash + source["is_auto_analyzed"] = test_item.isAutoAnalyzed + source["test_item_name"] = text_processing.preprocess_test_item_name(test_item.testItemName) + source["issue_type"] = transform_issue_type_into_lowercase(test_item.issueType) + source["start_time"] = datetime(*test_item.startTime[:6]).strftime("%Y-%m-%d %H:%M:%S") + return log_template + + +def _fill_common_fields(log_template: dict, log: Log, prepared_log: PreparedLogMessage) -> None: + log_template['_id'] = log.logId + source = compute_if_absent(log_template, '_source', {}) + source['cluster_id'] = str(log.clusterId) + source['cluster_message'] = log.clusterMessage + source['log_level'] = log.logLevel + source['original_message'] = log.message + source['message'] = prepared_log.message + source['message_lines'] = text_processing.calculate_line_number(prepared_log.clean_message) + source['message_words_number'] = len( + text_processing.split_words(prepared_log.clean_message, split_urls=False)) + source['stacktrace'] = prepared_log.stacktrace + source['potential_status_codes'] = prepared_log.exception_message_potential_status_codes + source['found_exceptions'] = prepared_log.exception_found + source['whole_message'] = '\n'.join([prepared_log.exception_message_no_params, prepared_log.stacktrace]) + + +def _fill_log_fields(log_template: dict, log: Log, number_of_lines: int) -> dict[str, Any]: + prepared_log = PreparedLogMessage(log.message, number_of_lines) + _fill_common_fields(log_template, log, prepared_log) + source = compute_if_absent(log_template, '_source', {}) + source["detected_message"] = prepared_log.exception_message_no_numbers + source["detected_message_with_numbers"] = prepared_log.exception_message + source["log_time"] = datetime(*log.logTime[:6]).strftime("%Y-%m-%d %H:%M:%S") + source["cluster_with_numbers"] = utils.extract_clustering_setting(log.clusterId) + source["only_numbers"] = prepared_log.exception_message_numbers + source["urls"] = prepared_log.exception_message_urls + source["paths"] = prepared_log.exception_message_paths + source["message_params"] = prepared_log.exception_message_params + source["found_exceptions_extended"] = prepared_log.exception_found_extended + source["detected_message_extended"] = \ + text_processing.enrich_text_with_method_and_classes(prepared_log.exception_message) + source["detected_message_without_params_extended"] = \ + text_processing.enrich_text_with_method_and_classes(prepared_log.exception_message_no_params) + source["stacktrace_extended"] = \ + text_processing.enrich_text_with_method_and_classes(prepared_log.stacktrace) + source["message_extended"] = \ + text_processing.enrich_text_with_method_and_classes(prepared_log.message) + source["message_without_params_extended"] = \ + text_processing.enrich_text_with_method_and_classes(prepared_log.message_no_params) + source["detected_message_without_params_and_brackets"] = \ + prepared_log.exception_message_no_params + source["message_without_params_and_brackets"] = prepared_log.message_no_params + source["found_tests_and_methods"] = prepared_log.test_and_methods_extended + + for field in ["message", "detected_message", "detected_message_with_numbers", + "stacktrace", "only_numbers", "found_exceptions", "found_exceptions_extended", + "detected_message_extended", "detected_message_without_params_extended", + "stacktrace_extended", "message_extended", "message_without_params_extended", + "detected_message_without_params_and_brackets", "message_without_params_and_brackets"]: + source[field] = text_processing.leave_only_unique_lines(source[field]) + source[field] = text_processing.clean_colon_stacking(source[field]) + return log_template + + +def prepare_log(launch: Launch, test_item: TestItem, log: Log, project: str) -> dict: + log_template = create_log_template() + log_template = _fill_launch_test_item_fields(log_template, launch, test_item, project) + log_template = _fill_log_fields(log_template, log, launch.analyzerConfig.numberOfLogLines) + return log_template + + +def _fill_test_item_info_fields(log_template: dict, test_item_info: TestItemInfo, project: str) -> dict[str, Any]: + log_template["_index"] = project + source = compute_if_absent(log_template, '_source', {}) + source["launch_id"] = test_item_info.launchId + source["launch_name"] = test_item_info.launchName + source["launch_number"] = getattr(test_item_info, 'launchNumber', 0) + source["test_item"] = test_item_info.testItemId + source["unique_id"] = test_item_info.uniqueId + source["test_case_hash"] = test_item_info.testCaseHash + source["test_item_name"] = text_processing.preprocess_test_item_name( + test_item_info.testItemName) + source["is_auto_analyzed"] = False + source["issue_type"] = "" + source["start_time"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + return log_template + + +def prepare_log_for_suggests(test_item_info: TestItemInfo, log: Log, project: str) -> dict: + log_template = create_log_template() + log_template = _fill_test_item_info_fields(log_template, test_item_info, project) + log_template = _fill_log_fields( + log_template, log, test_item_info.analyzerConfig.numberOfLogLines) + return log_template + + +def get_words_in_stacktrace(stacktrace: str) -> dict[str, int]: + words = {} + for word in text_processing.split_words(stacktrace): + if '.' in word and len(word.split('.')) > 2: + words[word] = 1 + return words + + +def prepare_log_words(launches: list[Launch]) -> tuple[dict[str, int], int]: + log_words = {} + project = None + for launch in launches: + project = launch.project + for test_item in launch.testItems: + for log in test_item.logs: + if log.logLevel < utils.ERROR_LOGGING_LEVEL or not log.message.strip(): + continue + cleaned_message = clean_message(log.message) + _, stacktrace = text_processing.detect_log_description_and_stacktrace(cleaned_message) + log_words.update(get_words_in_stacktrace(stacktrace)) + return log_words, project + + +def prepare_log_clustering_light(launch: Launch, test_item: TestItem, log: Log, project: str) -> dict[str, Any]: + log_template = create_log_template() + log_template = _fill_launch_test_item_fields(log_template, launch, test_item, project) + prepared_log = PreparedLogMessageClustering(log.message, -1) + _fill_common_fields(log_template, log, prepared_log) + return log_template + + +def prepare_logs_for_clustering(launch: Launch, project: str) -> list[list[dict[str, Any]]]: + log_messages = [] + for test_item in launch.testItems: + prepared_logs = [] + for log in test_item.logs: + if log.logLevel < utils.ERROR_LOGGING_LEVEL: + continue + prepared_logs.append(prepare_log_clustering_light(launch, test_item, log, project)) + log_messages.append(prepared_logs) + return log_messages diff --git a/app/commons/similarity_calculator.py b/app/commons/similarity_calculator.py index 7d5a8fad..194a2a4d 100644 --- a/app/commons/similarity_calculator.py +++ b/app/commons/similarity_calculator.py @@ -20,100 +20,36 @@ from app.machine_learning.models.weighted_similarity_calculator import WeightedSimilarityCalculator from app.utils import text_processing +FIELDS_MAPPING_FOR_WEIGHTING = { + 'message_without_params_extended': ['detected_message_without_params_extended', 'stacktrace_extended'], + 'message_extended': ['detected_message_extended', 'stacktrace_extended'] +} +ARTIFICIAL_COLUMNS = {'namespaces_stacktrace'} + + +def multiply_vectors_by_weight(rows, weights): + return np.dot(np.reshape(weights, [-1]), rows) + + +def normalize_weights(weights): + normalized_weights = np.asarray(weights) / np.min(weights) + return np.clip(normalized_weights, a_min=1.0, a_max=3.0) + class SimilarityCalculator: + __similarity_dict: dict[str, dict] similarity_model: WeightedSimilarityCalculator + config: dict[str, Any] + object_id_weights: dict[str, list[float]] def __init__(self, config: dict[str, Any], similarity_model: WeightedSimilarityCalculator): self.similarity_model = similarity_model self.config = config - self.similarity_dict = {} + self.__similarity_dict = {} self.object_id_weights = {} - self.fields_mapping_for_weighting = { - "message_without_params_extended": [ - "detected_message_without_params_extended", "stacktrace_extended"], - "message_extended": ["detected_message_extended", "stacktrace_extended"] - } - self.artificial_columns = ["namespaces_stacktrace"] - - def find_similarity(self, all_results: list[tuple[dict[str, Any], dict[str, Any]]], fields: list[str]) -> None: - for field in fields: - if field in self.similarity_dict: - continue - self.similarity_dict[field] = {} - log_field_ids: dict = {} - index_in_message_array = 0 - count_vector_matrix: np.ndarray | None = None - all_messages: list[str] = [] - all_messages_needs_reweighting: list[int] = [] - needs_reweighting_wc: bool = False - for log, res in all_results: - for obj in [log] + res["hits"]["hits"]: - if obj["_id"] not in log_field_ids: - if field not in self.artificial_columns and ( - field not in obj["_source"] or not obj["_source"][field].strip()): - log_field_ids[obj["_id"]] = -1 - else: - text = [] - needs_reweighting = 0 - if (self.config["number_of_log_lines"] == -1 - and field in self.fields_mapping_for_weighting): - fields_to_use = self.fields_mapping_for_weighting[field] - text = self.similarity_model.message_to_array( - obj["_source"][fields_to_use[0]], obj["_source"][fields_to_use[1]]) - elif field == "namespaces_stacktrace": - gathered_lines = [] - weights = [] - for line in obj["_source"]["stacktrace"].split("\n"): - line_words = text_processing.split_words( - line, min_word_length=self.config["min_word_length"]) - for word in line_words: - part_of_namespace = ".".join(word.split(".")[:2]) - if part_of_namespace in self.config["chosen_namespaces"]: - gathered_lines.append(" ".join(line_words)) - weights.append(self.config["chosen_namespaces"][part_of_namespace]) - if len(gathered_lines): - text = gathered_lines - self.object_id_weights[obj["_id"]] = weights - else: - text = [] - for line in obj["_source"]["stacktrace"].split("\n"): - text.append(" ".join(text_processing.split_words( - line, min_word_length=self.config["min_word_length"]))) - text = text_processing.filter_empty_lines(text) - self.object_id_weights[obj["_id"]] = [1] * len(text) - elif field.startswith("stacktrace"): - if text_processing.does_stacktrace_need_words_reweighting(obj["_source"][field]): - needs_reweighting = 1 - text = self.similarity_model.message_to_array("", obj["_source"][field]) - else: - text = [" ".join( - text_processing.split_words( - obj["_source"][field], min_word_length=self.config["min_word_length"]))] - if not text: - log_field_ids[obj["_id"]] = -1 - else: - all_messages.extend(text) - all_messages_needs_reweighting.append(needs_reweighting) - log_field_ids[obj["_id"]] = [index_in_message_array, len(all_messages) - 1] - index_in_message_array += len(text) - if all_messages: - needs_reweighting_wc = (all_messages_needs_reweighting - and sum(all_messages_needs_reweighting) == len(all_messages_needs_reweighting)) - vectorizer = CountVectorizer( - binary=not needs_reweighting_wc, analyzer="word", token_pattern="[^ ]+") - try: - count_vector_matrix = np.asarray(vectorizer.fit_transform(all_messages).toarray()) - except ValueError: - # All messages are empty or contains only stop words - pass - for log, res in all_results: - sim_dict = self._calculate_field_similarity( - log, res, log_field_ids, count_vector_matrix, needs_reweighting_wc, field) - for key, value in sim_dict.items(): - self.similarity_dict[field][key] = value - - def reweight_words_weights_by_summing(self, count_vector_matrix): + + @staticmethod + def reweight_words_weights_by_summing(count_vector_matrix): count_vector_matrix_weighted = np.zeros_like(count_vector_matrix, dtype=float) whole_sum_vector = np.sum(count_vector_matrix, axis=0) for i in range(len(count_vector_matrix)): @@ -124,40 +60,33 @@ def reweight_words_weights_by_summing(self, count_vector_matrix): count_vector_matrix_weighted[i][j] = count_vector_matrix[i][j] return count_vector_matrix_weighted - def multiply_vectors_by_weight(self, rows, weights): - return np.dot(np.reshape(weights, [-1]), rows) - - def normalize_weights(self, weights): - normalized_weights = np.asarray(weights) / np.min(weights) - return np.clip(normalized_weights, a_min=1.0, a_max=3.0) - def _calculate_field_similarity( self, log: dict, res: dict, log_field_ids: dict, count_vector_matrix: Optional[np.ndarray], - needs_reweighting_wc: bool, field: str) -> dict: + needs_reweighting_wc: bool, field: str) -> dict[tuple[str, str], dict[str, Any]]: all_results_similarity = {} for obj in res["hits"]["hits"]: - group_id = (obj["_id"], log["_id"]) + group_id = (str(obj["_id"]), str(log["_id"])) index_query_message = log_field_ids[log["_id"]] index_log_message = log_field_ids[obj["_id"]] if ((isinstance(index_query_message, int) and index_query_message < 0) and (isinstance(index_log_message, int) and index_log_message < 0)): all_results_similarity[group_id] = {"similarity": 1.0, "both_empty": True} elif ((isinstance(index_query_message, int) and index_query_message < 0) - or (isinstance(index_log_message, int) and index_log_message < 0)): + or (isinstance(index_log_message, int) and index_log_message < 0)): all_results_similarity[group_id] = {"similarity": 0.0, "both_empty": False} else: if count_vector_matrix is not None: query_vector = count_vector_matrix[index_query_message[0]:index_query_message[1] + 1] log_vector = count_vector_matrix[index_log_message[0]:index_log_message[1] + 1] if field == "namespaces_stacktrace": - query_vector = self.multiply_vectors_by_weight( - query_vector, self.normalize_weights(self.object_id_weights[log["_id"]])) - log_vector = self.multiply_vectors_by_weight( - log_vector, self.normalize_weights(self.object_id_weights[obj["_id"]])) + query_vector = multiply_vectors_by_weight( + query_vector, normalize_weights(self.object_id_weights[log["_id"]])) + log_vector = multiply_vectors_by_weight( + log_vector, normalize_weights(self.object_id_weights[obj["_id"]])) else: if needs_reweighting_wc: - query_vector = self.reweight_words_weights_by_summing(query_vector) - log_vector = self.reweight_words_weights_by_summing(log_vector) + query_vector = SimilarityCalculator.reweight_words_weights_by_summing(query_vector) + log_vector = SimilarityCalculator.reweight_words_weights_by_summing(log_vector) query_vector = self.similarity_model.weigh_data_rows(query_vector) log_vector = self.similarity_model.weigh_data_rows(log_vector) if needs_reweighting_wc: @@ -168,3 +97,118 @@ def _calculate_field_similarity( else: all_results_similarity[group_id] = {"similarity": 0.0, "both_empty": False} return all_results_similarity + + @staticmethod + def _create_count_vector_matrix(all_messages: list[str], + all_messages_needs_reweighting: bool) -> Optional[np.ndarray]: + count_vector_matrix: Optional[np.ndarray] = None + if all_messages: + vectorizer = CountVectorizer( + binary=not all_messages_needs_reweighting, analyzer="word", token_pattern="[^ ]+") + try: + count_vector_matrix = np.asarray(vectorizer.fit_transform(all_messages).toarray()) + except ValueError: + # All messages are empty or contains only stop words + pass + return count_vector_matrix + + def _calculate_similarity_for_all_results( + self, all_results: list[tuple[dict[str, Any], dict[str, Any]]], log_field_ids: dict, + all_messages: list[str], all_messages_needs_reweighting: bool, + field: str) -> dict[tuple[str, str], dict[str, Any]]: + count_vector_matrix = self._create_count_vector_matrix(all_messages, all_messages_needs_reweighting) + similarity = {} + for log, res in all_results: + sim_dict = self._calculate_field_similarity( + log, res, log_field_ids, count_vector_matrix, all_messages_needs_reweighting, field) + similarity.update(sim_dict) + return similarity + + def _process_weighted_field(self, obj: dict, field: str) -> list[str]: + fields_to_use = FIELDS_MAPPING_FOR_WEIGHTING[field] + return self.similarity_model.message_to_array( + obj["_source"][fields_to_use[0]], obj["_source"][fields_to_use[1]]) + + def _process_namespaces_stacktrace(self, obj: dict) -> list[str]: + gathered_lines = [] + weights = [] + for line in obj["_source"]["stacktrace"].split("\n"): + line_words = text_processing.split_words( + line, min_word_length=self.config["min_word_length"]) + for word in line_words: + part_of_namespace = ".".join(word.split(".")[:2]) + if part_of_namespace in self.config["chosen_namespaces"]: + gathered_lines.append(" ".join(line_words)) + weights.append(self.config["chosen_namespaces"][part_of_namespace]) + if len(gathered_lines): + self.object_id_weights[obj["_id"]] = weights + return gathered_lines + else: + text = [] + for line in obj["_source"]["stacktrace"].split("\n"): + text.append(" ".join(text_processing.split_words( + line, min_word_length=self.config["min_word_length"]))) + text = text_processing.filter_empty_lines(text) + self.object_id_weights[obj["_id"]] = [1] * len(text) + return text + + def _process_stacktrace_field(self, obj: dict, field: str) -> tuple[bool, list[str]]: + needs_reweighting = text_processing.does_stacktrace_need_words_reweighting(obj["_source"][field]) + text = self.similarity_model.message_to_array("", obj["_source"][field]) + return needs_reweighting, text + + def _process_generic_field(self, obj: dict, field: str) -> list[str]: + return [" ".join( + text_processing.split_words( + obj["_source"][field], min_word_length=self.config["min_word_length"]))] + + def _process_field(self, obj: dict, field: str) -> tuple[bool, list[str]]: + if self.config["number_of_log_lines"] == -1 and field in FIELDS_MAPPING_FOR_WEIGHTING: + return False, self._process_weighted_field(obj, field) + elif field == "namespaces_stacktrace": + return False, self._process_namespaces_stacktrace(obj) + elif field.startswith("stacktrace"): + return self._process_stacktrace_field(obj, field) + else: + return False, self._process_generic_field(obj, field) + + def _prepare_log_field_ids_and_messages(self, all_results: list[tuple[dict[str, Any], dict[str, Any]]], + field: str) -> tuple[dict[str, int], list[str], bool]: + log_field_ids: dict = {} + index_in_message_array = 0 + all_messages: list[str] = [] + all_messages_needs_reweighting: bool = True + for log, res in all_results: + for obj in [log] + res["hits"]["hits"]: + if obj["_id"] in log_field_ids: + continue + + if field not in ARTIFICIAL_COLUMNS and not obj['_source'].get(field, '').strip(): + log_field_ids[obj["_id"]] = -1 + continue + + needs_reweighting, text = self._process_field(obj, field) + if not text: + log_field_ids[obj["_id"]] = -1 + else: + all_messages.extend(text) + all_messages_needs_reweighting = all_messages_needs_reweighting and needs_reweighting + log_field_ids[obj["_id"]] = (index_in_message_array, len(all_messages) - 1) + index_in_message_array += len(text) + + return log_field_ids, all_messages, all_messages_needs_reweighting + + def _find_similarity_for_field(self, all_results: list[tuple[dict[str, Any], dict[str, Any]]], + field: str) -> dict[tuple[str, str], dict[str, Any]]: + log_field_ids, all_messages, all_messages_needs_reweighting = self._prepare_log_field_ids_and_messages( + all_results, field) + return self._calculate_similarity_for_all_results( + all_results, log_field_ids, all_messages, all_messages_needs_reweighting, field) + + def find_similarity(self, all_results: list[tuple[dict[str, Any], dict[str, Any]]], + fields: list[str]) -> dict[str, dict[tuple[str, str], dict[str, Any]]]: + for field in fields: + if field in self.__similarity_dict: + continue + self.__similarity_dict[field] = self._find_similarity_for_field(all_results, field) + return self.__similarity_dict diff --git a/app/machine_learning/boosting_featurizer.py b/app/machine_learning/boosting_featurizer.py index 1b060c72..07dcbbef 100644 --- a/app/machine_learning/boosting_featurizer.py +++ b/app/machine_learning/boosting_featurizer.py @@ -108,21 +108,20 @@ def __init__(self, results: list[tuple[dict[str, Any], dict[str, Any]]], config: fields_to_calc_similarity = self.find_columns_to_find_similarities_for() processed_results = self._perform_additional_text_processing(results) - if "filter_min_should_match" in self.config and len(self.config["filter_min_should_match"]) > 0: - self.similarity_calculator.find_similarity( - processed_results, self.config["filter_min_should_match"] + ["merged_small_logs"]) - for field in self.config["filter_min_should_match"]: + filter_min_should_match: Optional[list[str]] = self.config.get("filter_min_should_match", None) + if filter_min_should_match: + for field in filter_min_should_match: processed_results = self.filter_by_min_should_match(processed_results, field=field) - if "filter_min_should_match_any" in self.config and len(self.config["filter_min_should_match_any"]) > 0: - self.similarity_calculator.find_similarity( - processed_results, self.config["filter_min_should_match_any"] + ["merged_small_logs"]) + filter_min_should_match_any: Optional[list[str]] = self.config.get("filter_min_should_match_any", None) + if filter_min_should_match_any: processed_results = self.filter_by_min_should_match_any( - processed_results, fields=self.config["filter_min_should_match_any"]) + processed_results, fields=filter_min_should_match_any) self.test_item_log_stats = self._calculate_stats_by_test_item_ids(processed_results) - if "filter_by_all_logs_should_be_similar" in self.config: - if self.config["filter_by_all_logs_should_be_similar"]: - processed_results = self.filter_by_all_logs_should_be_similar(processed_results) - if "filter_by_test_case_hash" in self.config and self.config["filter_by_test_case_hash"]: + filter_by_all_logs_should_be_similar = self.config.get("filter_by_all_logs_should_be_similar", None) + if filter_by_all_logs_should_be_similar: + processed_results = self.filter_by_all_logs_should_be_similar(processed_results) + filter_by_test_case_hash = self.config.get("filter_by_test_case_hash", None) + if filter_by_test_case_hash: processed_results = self.filter_by_test_case_hash(processed_results) if "calculate_similarities" not in self.config or self.config["calculate_similarities"]: self.similarity_calculator.find_similarity(processed_results, fields_to_calc_similarity) @@ -259,8 +258,6 @@ def predict_particular_defect_type(self) -> dict[str, float]: mr_hit = search_rs["mrHit"] issue_type_to_compare: str = mr_hit["_source"]["issue_type"].lower() try: - if issue_type_to_compare.startswith('nd') or issue_type_to_compare.startswith('ti'): - continue res, res_prob = self.defect_type_predict_model.predict([det_message], issue_type_to_compare) result[issue_type] = res_prob[0][1] if len(res_prob[0]) == 2 else 0.0 self.used_model_info.update(self.defect_type_predict_model.get_model_info()) @@ -433,8 +430,8 @@ def is_only_merged_small_logs(self) -> dict[str, int]: scores_by_issue_type = self.find_most_relevant_by_type() similarity_percent_by_type = {} for issue_type, search_rs in scores_by_issue_type.items(): - group_id = (search_rs["mrHit"]["_id"], search_rs["compared_log"]["_id"]) - sim_obj = self.similarity_calculator.similarity_dict["message"][group_id] + group_id = (str(search_rs["mrHit"]["_id"]), str(search_rs["compared_log"]["_id"])) + sim_obj = self.similarity_calculator.find_similarity(self.raw_results, ["message"])["message"][group_id] similarity_percent_by_type[issue_type] = int(sim_obj["both_empty"]) return similarity_percent_by_type @@ -443,12 +440,14 @@ def filter_by_min_should_match(self, all_results: list[tuple[dict[str, Any], dic for log, res in all_results: new_elastic_res = [] for elastic_res in res["hits"]["hits"]: - group_id = (elastic_res["_id"], log["_id"]) - sim_obj = self.similarity_calculator.similarity_dict[field][group_id] + group_id = (str(elastic_res["_id"]), str(log["_id"])) + sim_dict = self.similarity_calculator.find_similarity(all_results, [field]) + sim_obj = sim_dict[field][group_id] similarity = sim_obj["similarity"] if sim_obj["both_empty"] and field in self.fields_to_replace_with_merged_logs: - sim_obj = self.similarity_calculator.similarity_dict["merged_small_logs"] - similarity = sim_obj[group_id]["similarity"] + sim_obj = self.similarity_calculator.find_similarity( + all_results, ['merged_small_logs'])['merged_small_logs'][group_id] + similarity = sim_obj["similarity"] if similarity >= self.config["min_should_match"]: new_elastic_res.append(elastic_res) new_results.append((log, {"hits": {"hits": new_elastic_res}})) @@ -463,14 +462,16 @@ def filter_by_min_should_match_any( for log, res in all_results: new_elastic_res = [] for elastic_res in res["hits"]["hits"]: - group_id = (elastic_res["_id"], log["_id"]) + group_id = (str(elastic_res["_id"]), str(log["_id"])) max_similarity = 0.0 + sim_dict = self.similarity_calculator.find_similarity(all_results, fields) for field in fields: - sim_obj = self.similarity_calculator.similarity_dict[field][group_id] + sim_obj = sim_dict[field][group_id] similarity = sim_obj["similarity"] if sim_obj["both_empty"] and field in self.fields_to_replace_with_merged_logs: - sim_obj = self.similarity_calculator.similarity_dict["merged_small_logs"] - similarity = sim_obj[group_id]["similarity"] + sim_obj = self.similarity_calculator.find_similarity( + all_results, ['merged_small_logs'])['merged_small_logs'][group_id] + similarity = sim_obj['similarity'] max_similarity = max(max_similarity, similarity) if max_similarity >= self.config["min_should_match"]: new_elastic_res.append(elastic_res) @@ -607,11 +608,11 @@ def _calculate_similarity_percent(self, field_name="message") -> dict[str, float :return: dict with issue type as key and float value as similarity percent """ scores_by_issue_type = self.find_most_relevant_by_type() - self.similarity_calculator.find_similarity(self.raw_results, [field_name]) + similarity_dict = self.similarity_calculator.find_similarity(self.raw_results, [field_name]) similarity_percent_by_type = {} for issue_type, search_rs in scores_by_issue_type.items(): - group_id = (search_rs["mrHit"]["_id"], search_rs["compared_log"]["_id"]) - sim_obj = self.similarity_calculator.similarity_dict[field_name][group_id] + group_id = (str(search_rs["mrHit"]["_id"]), str(search_rs["compared_log"]["_id"])) + sim_obj = similarity_dict[field_name][group_id] similarity_percent_by_type[issue_type] = sim_obj["similarity"] return similarity_percent_by_type @@ -652,10 +653,9 @@ def gather_features_info(self) -> tuple[list[list[float]], list[str]]: gathered_data_dict[feature] = [] for idx in sorted(issue_type_by_index.keys()): issue_type = issue_type_by_index[idx] - try: - _ = result[issue_type][0] + if isinstance(result[issue_type], list): gathered_data_dict[feature].append(result[issue_type]) - except: # noqa + else: gathered_data_dict[feature].append([round(result[issue_type], 2)]) self.previously_gathered_features[feature] = gathered_data_dict[feature] gathered_data = utils.gather_feature_list(gathered_data_dict, self.feature_ids) diff --git a/app/machine_learning/models/defect_type_model.py b/app/machine_learning/models/defect_type_model.py index aa93f92b..750dfa2f 100644 --- a/app/machine_learning/models/defect_type_model.py +++ b/app/machine_learning/models/defect_type_model.py @@ -14,9 +14,11 @@ import re from collections import Counter -from typing import Optional +from typing import Optional, Any +import numpy as np import pandas as pd +from scipy.sparse import csr_matrix from sklearn.ensemble import RandomForestClassifier from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.metrics import classification_report, confusion_matrix, f1_score @@ -30,34 +32,65 @@ LOGGER = logging.getLogger('analyzerApp.DefectTypeModel') MODEL_FILES: list[str] = ['count_vectorizer_models.pickle', 'models.pickle'] DATA_FIELD = 'detected_message_without_params_extended' -BASE_DEFECT_TYPE_PATTERN = re.compile(r'^([^_]+)_.*|^(\D+)\d+') +BASE_DEFECT_TYPE_PATTERN = re.compile(r'^(?:([^_]+)_\S+|(\D+)\d+)$') DEFAULT_N_ESTIMATORS = 10 -def get_model(self: DefaultDict, model_name: str): +# noinspection PyMethodMayBeStatic +class DummyVectorizer: + def transform(self, data: list[str]) -> csr_matrix: + return csr_matrix(np.zeros((len(data), 1))) + + def get_feature_names_out(self) -> list[str]: + return ['dummy'] + + +# noinspection PyMethodMayBeStatic +class DummyClassifier: + def predict(self, data: pd.DataFrame) -> np.ndarray: + return np.zeros(data.shape[0]) + + def predict_proba(self, data: pd.DataFrame) -> np.ndarray: + return np.zeros((data.shape[0], 2)) + + +DEFAULT_MODEL = DummyClassifier() +DEFAULT_VECTORIZER = DummyVectorizer() + + +def get_model(self: DefaultDict, model_name: str, default_value: any) -> Any: m = BASE_DEFECT_TYPE_PATTERN.match(model_name) if not m: raise KeyError(model_name) - base_model_name = m.group(1) - if not base_model_name: - base_model_name = m.group(2) + base_model_name = (m.group(1) or m.group(2)).strip() if not base_model_name: raise KeyError(model_name) - return self[base_model_name] + if base_model_name in self: + return self[base_model_name] + else: + return default_value + + +def get_vectorizer_model(self: Any, model_name: str) -> Any: + return get_model(self, model_name, DEFAULT_VECTORIZER) + + +def get_classifier_model(self: Any, model_name: str) -> Any: + return get_model(self, model_name, DEFAULT_MODEL) class DefectTypeModel(MlModel): _loaded: bool - count_vectorizer_models: DefaultDict[str, TfidfVectorizer] - models: DefaultDict[str, RandomForestClassifier] + count_vectorizer_models: DefaultDict[str, TfidfVectorizer | DummyVectorizer] + models: DefaultDict[str, RandomForestClassifier | DummyClassifier] n_estimators: int def __init__(self, object_saver: ObjectSaver, tags: str = 'global defect type model', *, n_estimators: Optional[int] = None) -> None: super().__init__(object_saver, tags) self._loaded = False - self.count_vectorizer_models = DefaultDict(get_model) - self.models = DefaultDict(get_model) + self.count_vectorizer_models = DefaultDict(get_vectorizer_model) + self.models = DefaultDict(get_classifier_model) self.n_estimators = n_estimators if n_estimators is not None else DEFAULT_N_ESTIMATORS @property @@ -68,8 +101,8 @@ def load_model(self) -> None: if self.loaded: return model = self._load_models(MODEL_FILES) - self.count_vectorizer_models = DefaultDict(get_model, **model[0]) - self.models = DefaultDict(get_model, **model[1]) + self.count_vectorizer_models = DefaultDict(get_vectorizer_model, **model[0]) + self.models = DefaultDict(get_classifier_model, **model[1]) self._loaded = True def save_model(self): diff --git a/app/service/analyzer_service.py b/app/service/analyzer_service.py index 5b20e39f..57ec079e 100644 --- a/app/service/analyzer_service.py +++ b/app/service/analyzer_service.py @@ -17,8 +17,6 @@ from app.commons import logging from app.commons.model.launch_objects import SearchConfig, Launch, TestItemInfo, AnalyzerConf from app.commons.model.ml import ModelInfo -from app.commons.log_merger import LogMerger -from app.commons.log_requests import LogRequests from app.commons.model_chooser import ModelChooser from app.utils import utils @@ -103,15 +101,11 @@ def add_constraints_for_launches_into_query_suggest(query: dict, test_item_info: class AnalyzerService: search_cfg: SearchConfig launch_boost: float - log_requests: LogRequests - log_merger: LogMerger model_chooser: ModelChooser def __init__(self, model_chooser: ModelChooser, search_cfg: SearchConfig): self.search_cfg = search_cfg self.launch_boost = abs(self.search_cfg.BoostLaunch) - self.log_requests = LogRequests() - self.log_merger = LogMerger() self.model_chooser = model_chooser def find_min_should_match_threshold(self, analyzer_config: AnalyzerConf): diff --git a/app/service/auto_analyzer_service.py b/app/service/auto_analyzer_service.py index 69a3d7c2..7d6b974e 100644 --- a/app/service/auto_analyzer_service.py +++ b/app/service/auto_analyzer_service.py @@ -19,10 +19,8 @@ from time import time, sleep from app.amqp.amqp import AmqpClient -from app.commons import logging -from app.commons import object_saving +from app.commons import logging, request_factory, object_saving, log_merger from app.commons.esclient import EsClient -from app.commons.log_requests import LogRequests from app.commons.model.launch_objects import AnalysisResult, BatchLogInfo, AnalysisCandidate, SuggestAnalysisResult, \ SearchConfig, ApplicationConfig, Launch from app.commons.model.ml import ModelType @@ -221,15 +219,14 @@ def leave_only_similar_logs(self, candidates_with_no_defect, boosting_config): boosting_config, similarity_model=self.similarity_model) if no_defect_candidate_exists: - _similarity_calculator.find_similarity( - [(log_info, search_res)], - ["message", "merged_small_logs"]) + sim_dict = _similarity_calculator.find_similarity( + [(log_info, search_res)], ["message", "merged_small_logs"]) for obj in search_res["hits"]["hits"]: - group_id = (obj["_id"], log_info["_id"]) - if group_id in _similarity_calculator.similarity_dict["message"]: - sim_val = _similarity_calculator.similarity_dict["message"][group_id] + group_id = (str(obj["_id"]), str(log_info["_id"])) + if group_id in sim_dict["message"]: + sim_val = sim_dict["message"][group_id] if sim_val["both_empty"]: - sim_val = _similarity_calculator.similarity_dict["merged_small_logs"][group_id] + sim_val = sim_dict["merged_small_logs"][group_id] threshold = boosting_config["min_should_match"] if not sim_val["both_empty"] and sim_val["similarity"] >= threshold: new_search_res.append(obj) @@ -337,9 +334,9 @@ def _query_elasticsearch(self, launches: list[Launch], max_batch_size=30): logger.info("Early finish from analyzer before timeout") break unique_logs = text_processing.leave_only_unique_logs(test_item.logs) - prepared_logs = [LogRequests._prepare_log(launch, test_item, log, index_name) + prepared_logs = [request_factory.prepare_log(launch, test_item, log, index_name) for log in unique_logs if log.logLevel >= utils.ERROR_LOGGING_LEVEL] - results, _ = self.log_merger.decompose_logs_merged_and_without_duplicates(prepared_logs) + results, _ = log_merger.decompose_logs_merged_and_without_duplicates(prepared_logs) for log in results: message = log["_source"]["message"].strip() diff --git a/app/service/cluster_service.py b/app/service/cluster_service.py index 8da7ffc4..10e8e19e 100644 --- a/app/service/cluster_service.py +++ b/app/service/cluster_service.py @@ -23,10 +23,8 @@ from sklearn.feature_extraction.text import CountVectorizer from app.amqp.amqp import AmqpClient -from app.commons import clusterizer, logging +from app.commons import clusterizer, logging, request_factory, log_merger from app.commons.esclient import EsClient -from app.commons.log_merger import LogMerger -from app.commons.log_requests import LogRequests from app.commons.model.launch_objects import (ClusterResult, ClusterInfo, SearchConfig, ApplicationConfig, LaunchInfoForClustering) from app.utils import utils, text_processing @@ -38,18 +36,15 @@ class ClusterService: app_config: ApplicationConfig search_cfg: SearchConfig es_client: EsClient - log_requests: LogRequests - log_merger: LogMerger def __init__(self, app_config: ApplicationConfig, search_cfg: SearchConfig): self.app_config = app_config self.search_cfg = search_cfg self.es_client = EsClient(app_config=self.app_config) - self.log_requests = LogRequests() - self.log_merger = LogMerger() - def add_query_with_start_time_decay(self, main_query: dict[str, Any]) -> dict[str, Any]: + def get_query_with_start_time_decay(self, main_query: dict[str, Any]) -> dict[str, Any]: return { + "_source": main_query["_source"], "size": main_query["size"], "query": { "function_score": { @@ -78,9 +73,8 @@ def build_search_similar_items_query( self, queried_log: dict[str, Any], message: str, launch_info: LaunchInfoForClustering, min_should_match: str = "95%") -> dict[str, Any]: """Build search query""" - query = { - "_source": ["whole_message", "test_item", "is_merged", - "detected_message", "stacktrace", "launch_id", "cluster_id", + query: dict[str, Any] = { + "_source": ["whole_message", "test_item", "is_merged", "detected_message", "launch_id", "cluster_id", "cluster_message", "potential_status_codes", "found_exceptions"], "size": 10, "query": { @@ -129,7 +123,7 @@ def build_search_similar_items_query( ) utils.append_potential_status_codes( query, queried_log, boost=1.0, max_query_terms=self.search_cfg.MaxQueryTerms) - return self.add_query_with_start_time_decay(query) + return self.get_query_with_start_time_decay(query) def find_similar_items_from_es(self, groups: dict[int, list[int]], log_dict: dict[int, dict[str, Any]], log_messages: list[str], log_ids: set[str], launch_info: LaunchInfoForClustering, @@ -197,7 +191,7 @@ def find_similar_items_from_es(self, groups: dict[int, list[int]], log_dict: dic if log_dict_part[ind]["_source"]["launch_id"] != launch_info.launch.launchId: continue log_ids.add(str(log_dict_part[ind]["_id"])) - new_group_log_ids.append(str(log_dict_part[ind]["_id"])) + new_group_log_ids.append(log_dict_part[ind]["_id"]) new_test_items.add(int(log_dict_part[ind]["_source"]["test_item"])) if not cluster_id or not cluster_message: continue @@ -235,12 +229,11 @@ def regroup_by_error_and_status_codes( def cluster_messages_with_grouping_by_error( self, log_messages: list[str], log_dict: dict[int, dict[str, Any]], unique_errors_min_should_match: float) -> dict[int, list[int]]: - regroupped_by_error = self.regroup_by_error_and_status_codes( - log_messages, log_dict) + regrouped_by_error = self.regroup_by_error_and_status_codes(log_messages, log_dict) _clusterizer = clusterizer.Clusterizer() - all_groups = {} + all_groups: dict[int, list[int]] = {} start_group_id = 0 - for group in regroupped_by_error.values(): + for group in regrouped_by_error.values(): log_messages_part = [] log_messages_idx_dict = {} for i, idx in enumerate(group): @@ -292,7 +285,7 @@ def gather_cluster_results( log_ids_for_merged_logs: dict[str, list[int]], launch_info: LaunchInfoForClustering) -> tuple[list[ClusterInfo], int, dict[str, tuple[int, str]]]: merged_logs_to_update = {} - clusters_found = {} + clusters_found: dict[int, tuple[list[int], list[int]]] = {} cluster_message_by_id = {} for group in groups: cnt_items = len(groups[group]) @@ -357,8 +350,9 @@ def find_clusters(self, launch_info: LaunchInfoForClustering): log_ids = {} try: unique_errors_min_should_match = launch_info.launch.analyzerConfig.uniqueErrorsMinShouldMatch / 100.0 # noqa - log_messages, log_dict, log_ids_for_merged_logs = self.log_requests.prepare_logs_for_clustering( # noqa - launch_info.launch, launch_info.numberOfLogLines, launch_info.cleanNumbers, index_name) + prepared_logs = request_factory.prepare_logs_for_clustering(launch_info.launch, index_name) + log_messages, log_dict, log_ids_for_merged_logs = log_merger.merge_logs( + prepared_logs, launch_info.numberOfLogLines, launch_info.cleanNumbers) log_ids = set([str(log["_id"]) for log in log_dict.values()]) groups = self.cluster_messages_with_grouping_by_error( log_messages, log_dict, unique_errors_min_should_match) @@ -414,14 +408,6 @@ def find_clusters(self, launch_info: LaunchInfoForClustering): logger.debug("Stats info %s", results_to_share) logger.info("Processed the launch. It took %.2f sec.", time() - t_start) logger.info("Finished clustering for the launch with %d clusters.", cluster_num) - for cluster in clusters: - # Set original messages for clusters to show in UI - log_ids = set(cluster.logIds) - for test_item in launch_info.launch.testItems: - for log in test_item.logs: - if log.logId in log_ids: - cluster.clusterMessage = log.message - break return ClusterResult( project=launch_info.project, launchId=launch_info.launch.launchId, diff --git a/app/service/namespace_finder_service.py b/app/service/namespace_finder_service.py index f0e27929..414ce984 100644 --- a/app/service/namespace_finder_service.py +++ b/app/service/namespace_finder_service.py @@ -14,9 +14,8 @@ from time import time -from app.commons import logging, namespace_finder +from app.commons import logging, namespace_finder, request_factory from app.commons.model.launch_objects import ApplicationConfig, Launch -from app.commons.log_requests import LogRequests from app.utils import utils logger = logging.getLogger("analyzerApp.namespaceFinderService") @@ -24,17 +23,15 @@ class NamespaceFinderService: namespace_finder: namespace_finder.NamespaceFinder - log_requests: LogRequests def __init__(self, app_config: ApplicationConfig): self.namespace_finder = namespace_finder.NamespaceFinder(app_config) - self.log_requests = LogRequests() @utils.ignore_warnings def update_chosen_namespaces(self, launches: list[Launch]): logger.info("Started updating chosen namespaces") t_start = time() - log_words, project_id = LogRequests.prepare_log_words(launches) + log_words, project_id = request_factory.prepare_log_words(launches) logger.debug(f'Project id {project_id}') if project_id is not None: self.namespace_finder.update_namespaces(project_id, log_words) diff --git a/app/service/search_service.py b/app/service/search_service.py index ede9f862..43e20fcd 100644 --- a/app/service/search_service.py +++ b/app/service/search_service.py @@ -17,11 +17,9 @@ import elasticsearch import elasticsearch.helpers -from app.commons import logging, similarity_calculator, object_saving +from app.commons import logging, similarity_calculator, object_saving, request_factory, log_merger from app.commons.esclient import EsClient from app.commons.model.launch_objects import SearchLogInfo, Log, SearchConfig, ApplicationConfig -from app.commons.log_merger import LogMerger -from app.commons.log_requests import LogRequests, create_log_template from app.machine_learning.models.weighted_similarity_calculator import WeightedSimilarityCalculator from app.utils import utils, text_processing @@ -32,16 +30,12 @@ class SearchService: app_config: ApplicationConfig search_cfg: SearchConfig es_client: EsClient - log_requests: LogRequests - log_merger: LogMerger similarity_model: WeightedSimilarityCalculator def __init__(self, app_config: ApplicationConfig, search_cfg: SearchConfig): self.app_config = app_config self.search_cfg = search_cfg self.es_client = EsClient(app_config=self.app_config) - self.log_requests = LogRequests() - self.log_merger = LogMerger() if not self.search_cfg.SimilarityWeightsFolder: raise ValueError('SimilarityWeightsFolder is not set') self.similarity_model = ( @@ -161,9 +155,9 @@ def prepare_messages_for_queries(self, search_req): if not message.strip(): continue - queried_log = create_log_template() - queried_log = LogRequests._fill_log_fields(queried_log, Log(logId=global_id, message=message), - search_req.logLines) + queried_log = request_factory.create_log_template() + queried_log = request_factory._fill_log_fields( + queried_log, Log(logId=global_id, message=message), search_req.logLines) msg_words = " ".join(text_processing.split_words(queried_log["_source"]["message"])) if not msg_words.strip() or msg_words in searched_logs: @@ -172,7 +166,7 @@ def prepare_messages_for_queries(self, search_req): logs_to_query.append(queried_log) global_id += 1 - logs_to_query, _ = self.log_merger.decompose_logs_merged_and_without_duplicates(logs_to_query) + logs_to_query, _ = log_merger.decompose_logs_merged_and_without_duplicates(logs_to_query) return logs_to_query def search_similar_items_for_log(self, search_req, queried_log, @@ -226,19 +220,19 @@ def search_logs(self, search_req): "number_of_log_lines": search_req.logLines }, similarity_model=self.similarity_model) - _similarity_calculator.find_similarity( + sim_dict = _similarity_calculator.find_similarity( [(queried_log, search_results)], ["message", "potential_status_codes", "merged_small_logs"]) - for group_id, similarity_obj in _similarity_calculator.similarity_dict["message"].items(): + for group_id, similarity_obj in sim_dict["message"].items(): log_id, _ = group_id similarity_percent = similarity_obj["similarity"] if similarity_obj["both_empty"]: - similarity_obj = _similarity_calculator.similarity_dict["merged_small_logs"][group_id] + similarity_obj = sim_dict["merged_small_logs"][group_id] similarity_percent = similarity_obj["similarity"] logger.debug("Log with id %s has %.3f similarity with the queried log '%s'", log_id, similarity_percent, message_to_use) potential_status_codes_match = 0.0 - _similarity_dict = _similarity_calculator.similarity_dict["potential_status_codes"] + _similarity_dict = sim_dict["potential_status_codes"] if group_id in _similarity_dict: potential_status_codes_match = _similarity_dict[group_id]["similarity"] if potential_status_codes_match < 0.99: diff --git a/app/service/suggest_service.py b/app/service/suggest_service.py index e3523f25..2eb905bf 100644 --- a/app/service/suggest_service.py +++ b/app/service/suggest_service.py @@ -20,9 +20,8 @@ import elasticsearch.helpers from app.amqp.amqp import AmqpClient -from app.commons import logging, similarity_calculator, object_saving +from app.commons import logging, similarity_calculator, object_saving, request_factory, log_merger from app.commons.esclient import EsClient -from app.commons.log_requests import LogRequests from app.commons.model.launch_objects import SuggestAnalysisResult, SearchConfig, ApplicationConfig, TestItemInfo, \ AnalyzerConf from app.commons.model.ml import ModelType, TrainInfo @@ -210,9 +209,8 @@ def deduplicate_results(self, gathered_results, scores_by_test_items, test_item_ items_to_compare = {"hits": {"hits": [scores_by_test_items[test_item_id_first]["mrHit"]]}} all_pairs_to_check.append((scores_by_test_items[test_item_id_second]["mrHit"], items_to_compare)) - _similarity_calculator.find_similarity( - all_pairs_to_check, - ["detected_message_with_numbers", "stacktrace", "merged_small_logs"]) + sim_dict = _similarity_calculator.find_similarity( + all_pairs_to_check, ["detected_message_with_numbers", "stacktrace", "merged_small_logs"]) filtered_results = [] deleted_indices = set() @@ -222,14 +220,14 @@ def deduplicate_results(self, gathered_results, scores_by_test_items, test_item_ for j in range(i + 1, len(gathered_results)): test_item_id_first = test_item_ids[gathered_results[i][0]] test_item_id_second = test_item_ids[gathered_results[j][0]] - group_id = (scores_by_test_items[test_item_id_first]["mrHit"]["_id"], - scores_by_test_items[test_item_id_second]["mrHit"]["_id"]) - if group_id not in _similarity_calculator.similarity_dict["detected_message_with_numbers"]: + group_id = (str(scores_by_test_items[test_item_id_first]["mrHit"]["_id"]), + str(scores_by_test_items[test_item_id_second]["mrHit"]["_id"])) + if group_id not in sim_dict["detected_message_with_numbers"]: continue - det_message = _similarity_calculator.similarity_dict["detected_message_with_numbers"] + det_message = sim_dict["detected_message_with_numbers"] detected_message_sim = det_message[group_id] - stacktrace_sim = _similarity_calculator.similarity_dict["stacktrace"][group_id] - merged_logs_sim = _similarity_calculator.similarity_dict["merged_small_logs"][group_id] + stacktrace_sim = sim_dict["stacktrace"][group_id] + merged_logs_sim = sim_dict["merged_small_logs"][group_id] if detected_message_sim["similarity"] >= 0.98 and \ stacktrace_sim["similarity"] >= 0.98 and merged_logs_sim["similarity"] >= 0.98: deleted_indices.add(j) @@ -330,9 +328,9 @@ def prepare_logs_for_suggestions(self, test_item_info: TestItemInfo, index_name: prepared_logs, test_item_id_for_suggest = self.query_logs_for_cluster(test_item_info, index_name) else: unique_logs = text_processing.leave_only_unique_logs(test_item_info.logs) - prepared_logs = [LogRequests._prepare_log_for_suggests(test_item_info, log, index_name) + prepared_logs = [request_factory.prepare_log_for_suggests(test_item_info, log, index_name) for log in unique_logs if log.logLevel >= utils.ERROR_LOGGING_LEVEL] - logs, _ = self.log_merger.decompose_logs_merged_and_without_duplicates(prepared_logs) + logs, _ = log_merger.decompose_logs_merged_and_without_duplicates(prepared_logs) return logs, test_item_id_for_suggest def suggest_items(self, test_item_info: TestItemInfo): diff --git a/app/utils/log_preparation.py b/app/utils/log_preparation.py index 09ced684..ad9affc1 100644 --- a/app/utils/log_preparation.py +++ b/app/utils/log_preparation.py @@ -28,9 +28,13 @@ def basic_prepare(message: str) -> str: # This should go right after starting garbage clean-up cleaned_message = text_processing.unify_line_endings(cleaned_message) - cleaned_message = text_processing.remove_markdown_mode(cleaned_message) - cleaned_message = text_processing.replace_code_separators(cleaned_message) + cleaned_message = text_processing.delete_empty_lines(cleaned_message) + return cleaned_message + + +def clean_message(basic_message: str) -> str: + cleaned_message = text_processing.replace_code_separators(basic_message) cleaned_message = text_processing.remove_webdriver_auxiliary_info(cleaned_message) cleaned_message = text_processing.replace_tabs_for_newlines(cleaned_message) cleaned_message = text_processing.fix_big_encoded_urls(cleaned_message) @@ -43,28 +47,38 @@ def basic_prepare(message: str) -> str: return cleaned_message -def prepare_message(clean_message: str, number_of_lines: int, test_and_methods: set[str]) -> str: - message = text_processing.first_lines(clean_message, number_of_lines) - message = text_processing.replace_text_pieces(message, test_and_methods) - message = text_processing.delete_empty_lines(text_processing.remove_numbers(message)) - return message +def prepare_message(message: str, number_of_lines: int, test_and_methods: set[str]) -> str: + cleaned_message = text_processing.first_lines(message, number_of_lines) + cleaned_message = text_processing.replace_text_pieces(cleaned_message, test_and_methods) + return cleaned_message + + +def prepare_message_no_numbers(message: str, number_of_lines: int, test_and_methods: set[str]) -> str: + cleaned_message = prepare_message(message, number_of_lines, test_and_methods) + cleaned_message = text_processing.delete_empty_lines(text_processing.remove_numbers(cleaned_message)) + return cleaned_message def prepare_message_no_params(message: str) -> str: - message_without_params = text_processing.remove_numbers(message) - message_without_params = text_processing.clean_from_params(message_without_params) - return message_without_params + cleaned_message = text_processing.clean_from_params(message) + return cleaned_message -def prepare_exception_message_and_stacktrace(clean_message: str) -> tuple[str, str]: - exception_message, stacktrace = text_processing.detect_log_description_and_stacktrace(clean_message) +def prepare_exception_message_and_stacktrace(message: str) -> tuple[str, str]: + exception_message, stacktrace = text_processing.detect_log_description_and_stacktrace(message) stacktrace = text_processing.clean_from_brackets(stacktrace) stacktrace = text_processing.remove_numbers(stacktrace) return exception_message, stacktrace def prepare_exception_message_no_params(exception_message: str) -> str: - result = text_processing.remove_numbers(exception_message) - result = text_processing.clean_from_params(result) - result = text_processing.unify_spaces(result) - return result + cleaned_message = text_processing.clean_from_params(exception_message) + cleaned_message = text_processing.unify_spaces(cleaned_message) + return cleaned_message + + +def prepare_exception_message_no_params_no_numbers(exception_message: str) -> str: + cleaned_message = text_processing.remove_numbers(exception_message) + cleaned_message = text_processing.clean_from_params(cleaned_message) + cleaned_message = text_processing.unify_spaces(cleaned_message) + return cleaned_message diff --git a/app/utils/utils.py b/app/utils/utils.py index 846118cd..e2557578 100644 --- a/app/utils/utils.py +++ b/app/utils/utils.py @@ -350,3 +350,13 @@ def create_path(query: dict, path: tuple[str, ...], value: Any) -> Any: if last_element not in current_node: current_node[last_element] = value return current_node[last_element] + + +def compute_if_absent(on: dict[str, Any], key: str, default_value: Any) -> Any: + """Compute value for key in dictionary if it is absent. + + It is here just to mute SonarLint warning about possible KeyError. + """ + if key not in on: + on[key] = default_value + return on[key] diff --git a/requirements.txt b/requirements.txt index fc17b34d..e979ed13 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ pydantic==1.10.13 elasticsearch==7.0.0 requests==2.32.2 bump2version==1.0.1 -nltk==3.8.1 +nltk==3.9.1 scikit-learn==1.5.0 numpy==1.23.5 scipy==1.10.1 @@ -19,5 +19,5 @@ gensim==4.3.2 pandas==1.5.3 imbalanced-learn==0.12.3 urllib3==1.26.19 -certifi>=2023.7.22 +certifi>=2024.8.30 werkzeug>=3.0.6 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/requirements_windows.txt b/requirements_windows.txt index 965cb5bd..56ed1262 100644 --- a/requirements_windows.txt +++ b/requirements_windows.txt @@ -8,7 +8,7 @@ requests==2.32.0 assertpy==1.1 bump2version==1.0.1 flake8==5.0.4 -nltk==3.8.1 +nltk==3.9.1 scikit-learn==1.5.0 numpy==1.23.5 scipy==1.10.1 diff --git a/test/service/test_cluster_service.py b/test/service/test_cluster_service.py index 0fb37d9f..c2155b16 100644 --- a/test/service/test_cluster_service.py +++ b/test/service/test_cluster_service.py @@ -121,12 +121,12 @@ def test_find_clusters(self): clusters=[ launch_objects.ClusterInfo( clusterId=21874152824769751, - clusterMessage="error occurred \n error found \n error mined", + clusterMessage="error occurred\nerror found\nerror mined", logIds=[4, 5], itemIds=[2, 5]), launch_objects.ClusterInfo( clusterId=44972330576749361, - clusterMessage="error occurred \n error found \n assert query", + clusterMessage="error occurred\nerror found\nassert query", logIds=[9], itemIds=[6]) ]) @@ -162,8 +162,8 @@ def test_find_clusters(self): launchId=1, clusters=[ launch_objects.ClusterInfo( - clusterId="48859729558090231", - clusterMessage="error occurred \n error found \n assert query", + clusterId=48859729558090231, + clusterMessage="error occurred\nerror found", logIds=[4, 5, 9], itemIds=[2, 5, 6]) ]) @@ -205,13 +205,13 @@ def test_find_clusters(self): launchId=1, clusters=[ launch_objects.ClusterInfo( - clusterId="21874152824769751", - clusterMessage="error occurred \n error found \n error mined", + clusterId=21874152824769751, + clusterMessage="error occurred\nerror found\nerror mined", logIds=[4, 5], itemIds=[2, 5]), launch_objects.ClusterInfo( - clusterId="44972330576749361", - clusterMessage="error occurred \n error found \n assert query", + clusterId=44972330576749361, + clusterMessage="error occurred\nerror found\nassert query", logIds=[9], itemIds=[6]), ]) @@ -254,13 +254,13 @@ def test_find_clusters(self): launchId=1, clusters=[ launch_objects.ClusterInfo( - clusterId="123", + clusterId=123, clusterMessage="error occurred \n error found \n error mined", logIds=[4, 5, 111], itemIds=[2, 5]), launch_objects.ClusterInfo( - clusterId="44972330576749361", - clusterMessage="error occurred \n error found \n assert query", + clusterId=44972330576749361, + clusterMessage="error occurred\nerror found\nassert query", logIds=[9], itemIds=[6]) ]) @@ -296,8 +296,8 @@ def test_find_clusters(self): launchId=1, clusters=[ launch_objects.ClusterInfo( - clusterId="48859729558090231", - clusterMessage="error occurred \n error found \n assert query", + clusterId=48859729558090231, + clusterMessage="error occurred\nerror found", logIds=[4, 5, 9], itemIds=[2, 5, 6]) ]) @@ -334,8 +334,8 @@ def test_find_clusters(self): launchId=1, clusters=[ launch_objects.ClusterInfo( - clusterId="48859729558090231", - clusterMessage="error occurred \n error found \n assert query", + clusterId=48859729558090231, + clusterMessage="error occurred\nerror found", logIds=[4, 5, 9], itemIds=[2, 5, 6]) ]) @@ -387,19 +387,18 @@ def test_find_clusters(self): launchId=1, clusters=[ launch_objects.ClusterInfo( - clusterId="37711525315085941", - clusterMessage="AssertionError error occurred \n error found \n error mined", + clusterId=37711525315085941, + clusterMessage="AssertionError error occurred\nerror found", logIds=[4], itemIds=[2]), launch_objects.ClusterInfo( - clusterId="48851059259117511", - clusterMessage="AssertionError status code: 500 error occurred \n error found \n error " - "mined", + clusterId=83179189436345941, + clusterMessage="AssertionError status code 500 error occurred\nerror found", logIds=[5], itemIds=[5]), launch_objects.ClusterInfo( - clusterId="90988898127574211", - clusterMessage="NoSuchElementException error occurred \n error found \n assert query", + clusterId=90988898127574211, + clusterMessage="NoSuchElementException error occurred\nerror found", logIds=[9], itemIds=[6]), ]) @@ -449,19 +448,19 @@ def test_find_clusters(self): launchId=1, clusters=[ launch_objects.ClusterInfo( - clusterId="60604459849884091", - clusterMessage="error occurred twice", + clusterId=60604459849884091, + clusterMessage="error occurred twice\nAssertionError error occurred\nerror found", # noqa logIds=[3, 4], itemIds=[2]), launch_objects.ClusterInfo( - clusterId="9398573272102061", - clusterMessage="AssertionError status code: 500 error occurred", + clusterId=58202398056526781, + clusterMessage="AssertionError status code 500 error occurred", logIds=[5], itemIds=[5]), launch_objects.ClusterInfo( - clusterId="86465058569810291", - clusterMessage="NoSuchElementException error occurred \n error found \n assert query", + clusterId=86465058569810291, + clusterMessage="NoSuchElementException error occurred\nerror found\nassert query", # noqa logIds=[9], itemIds=[6]), @@ -476,8 +475,7 @@ def test_find_clusters(self): app_config = self.app_config if "app_config" in test: app_config = test["app_config"] - _cluster_service = ClusterService(app_config=app_config, - search_cfg=config) + _cluster_service = ClusterService(app_config=app_config, search_cfg=config) response = _cluster_service.find_clusters(test["launch_info"]) diff --git a/test/unit/commons/object_saving/test_minio_client.py b/test/unit/commons/object_saving/test_minio_client.py index 3fa957b1..e90453da 100644 --- a/test/unit/commons/object_saving/test_minio_client.py +++ b/test/unit/commons/object_saving/test_minio_client.py @@ -36,21 +36,27 @@ def run_s3(): server.stop() -def create_storage_client(): - return MinioClient(ApplicationConfig(minioHost=SERVER_HOST, minioRegion=REGION, minioBucketPrefix=BUCKET_PREFIX, +def create_storage_client(bucket_prefix=BUCKET_PREFIX): + return MinioClient(ApplicationConfig(minioHost=SERVER_HOST, minioRegion=REGION, minioBucketPrefix=bucket_prefix, minioAccessKey='minio', minioSecretKey='minio', minioUseTls=False)) -def test_object_not_exists(): - object_name = f'{random_alphanumeric(16)}.pickle' - minio_client = create_storage_client() +@pytest.mark.parametrize('bucket_prefix, bucket, object_name', [ + (BUCKET_PREFIX, '2', f'{random_alphanumeric(16)}.pickle'), + (f'test/{BUCKET_PREFIX}', '2', f'{random_alphanumeric(16)}.json'), +]) +def test_object_not_exists(bucket_prefix, bucket, object_name): + minio_client = create_storage_client(bucket_prefix) assert not minio_client.does_object_exists('2', object_name) -def test_object_exists(): - object_name = f'{random_alphanumeric(16)}.pickle' - minio_client = create_storage_client() +@pytest.mark.parametrize('bucket_prefix, bucket, object_name', [ + (BUCKET_PREFIX, '2', f'{random_alphanumeric(16)}.pickle'), + (f'test/{BUCKET_PREFIX}', '2', f'{random_alphanumeric(16)}.json'), +]) +def test_object_exists(bucket_prefix, bucket, object_name): + minio_client = create_storage_client(bucket_prefix) minio_client.put_project_object({'test': True}, '2', object_name) @@ -59,13 +65,15 @@ def test_object_exists(): def get_url(bucket, object_name): # noinspection HttpUrlsUsage - return f'http://{SERVER_HOST}/{BUCKET_PREFIX}{bucket}/{object_name}' + return f'http://{SERVER_HOST}/{bucket}/{object_name}' -def test_json_write(): - bucket = '2' - object_name = 'SIED2wqgAppe4XPl.json' - minio_client = create_storage_client() +@pytest.mark.parametrize('bucket_prefix, bucket, bucket_name, object_name, object_path', [ + (BUCKET_PREFIX, '2', f'{BUCKET_PREFIX}2', 'my_test_file_write.json', 'my_test_file_write.json',), + (f'test/{BUCKET_PREFIX}', '2', 'test', 'my_test_file_write.json', f'{BUCKET_PREFIX}2/my_test_file_write.json',), +]) +def test_json_write(bucket_prefix, bucket, bucket_name, object_name, object_path): + minio_client = create_storage_client(bucket_prefix) minio_client.put_project_object({'test': True}, bucket, object_name, using_json=True) @@ -76,14 +84,16 @@ def test_json_write(): 'SignedHeaders=host;user-agent;x-amz-content-sha256;x-amz-date, ' 'Signature=dc971726ff2b266f208b250089b2ba0be86352efad2858145b33c2ae085e7d71' } - response = requests.get(get_url(bucket, object_name), headers=headers) + response = requests.get(get_url(bucket_name, object_path), headers=headers) assert response.text == '{"test": true}' -def test_json_read(): - bucket = '2' - object_name = '5ymFfxpAOK2eKYxx.json' - minio_client = create_storage_client() +@pytest.mark.parametrize('bucket_prefix, bucket, bucket_name, object_name, object_path', [ + (BUCKET_PREFIX, '2', f'{BUCKET_PREFIX}2', 'my_test_file_read.json', 'my_test_file_read.json',), + (f'test/{BUCKET_PREFIX}', '2', 'test', 'my_test_file_read.json', f'{BUCKET_PREFIX}2/my_test_file_read.json',), +]) +def test_json_read(bucket_prefix, bucket, bucket_name, object_name, object_path): + minio_client = create_storage_client(bucket_prefix) headers = { 'x-amz-date': '20231124T124147Z', @@ -92,36 +102,65 @@ def test_json_read(): 'SignedHeaders=content-length;content-type;host;user-agent;x-amz-content-sha256;x-amz-date, ' 'Signature=d592f084a4f9fd46a8624a37323b5be843120bd9e7c075c925faea573f00511e' } - requests.put(get_url(bucket, object_name), headers=headers, data='{"test": true}'.encode('utf-8')) + requests.put(get_url(bucket_name, object_path), headers=headers, data='{"test": true}'.encode('utf-8')) result = minio_client.get_project_object(bucket, object_name, using_json=True) assert isinstance(result, dict) assert result['test'] is True -def test_not_existing_file_get(): - object_name = f'{random_alphanumeric(16)}.json' - minio_client = create_storage_client() +@pytest.mark.parametrize('bucket_prefix, bucket, bucket_name, object_name, object_path', [ + (BUCKET_PREFIX, '2', f'{BUCKET_PREFIX}2', 'my_test_file.json', 'my_test_file.json',), + (f'test/{BUCKET_PREFIX}', '2', 'test', 'my_test_file.json', f'{BUCKET_PREFIX}2/my_test_file.json',), +]) +def test_not_existing_file_get(bucket_prefix, bucket, bucket_name, object_name, object_path): + minio_client = create_storage_client(bucket_prefix) with pytest.raises(ValueError) as exc: minio_client.get_project_object('2', object_name) - assert exc.value.args[0] == f'Unable to get file: {object_name}' + assert exc.value.args[0] == f'Unable to get file in bucket "{bucket_name}" with path "{object_path}"' -def test_remove_not_existing_folder(): - path = 'test' +@pytest.mark.parametrize('bucket', ['2', '3']) +def test_remove_not_existing_folder(bucket): + path = 'test-remove-not-existing' minio_client = create_storage_client() - assert not minio_client.remove_folder_objects('3', path) + assert not minio_client.remove_folder_objects(bucket, path) -def test_remove_existing_folder(): - bucket = '5' - object_name = f'{random_alphanumeric(16)}.json' - path = 'test' +@pytest.mark.parametrize('bucket_prefix, bucket, bucket_name, object_name, path, object_path', [ + (BUCKET_PREFIX, '5', f'{BUCKET_PREFIX}5', 'my_test_file_folder.json', 'folder', 'folder/my_test_file_folder.json'), + (f'test/{BUCKET_PREFIX}', '5', 'test', 'my_test_file_folder.json', 'folder', + f'{BUCKET_PREFIX}5/folder/my_test_file_folder.json'), +]) +def test_get_existing_folder(bucket_prefix, bucket, bucket_name, object_name, path, object_path): resource = '/'.join([path, object_name]) - minio_client = create_storage_client() + minio_client = create_storage_client(bucket_prefix) + minio_client.put_project_object({'test': True}, bucket, resource) + + headers = { + 'x-amz-date': '20231124T123217Z', + 'x-amz-content-sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', + 'authorization': 'AWS4-HMAC-SHA256 Credential=minio/20231124/us-west-1/s3/aws4_request, ' + 'SignedHeaders=host;user-agent;x-amz-content-sha256;x-amz-date, ' + 'Signature=dc971726ff2b266f208b250089b2ba0be86352efad2858145b33c2ae085e7d71' + } + resource_url = get_url(bucket_name, object_path) + response = requests.get(resource_url, headers=headers) + assert response.status_code == 200, f'Failed to get file from {resource_url}' + + +@pytest.mark.parametrize('bucket_prefix, bucket, bucket_name, object_name, path, object_path', [ + (BUCKET_PREFIX, '5', f'{BUCKET_PREFIX}5', 'my_test_file_folder.json', 'folder', 'folder/my_test_file_folder.json'), + (f'test/{BUCKET_PREFIX}', '5', 'test', 'my_test_file_folder.json', 'folder', + f'{BUCKET_PREFIX}5/folder/my_test_file_folder.json'), +]) +def test_remove_existing_folder(bucket_prefix, bucket, bucket_name, object_name, path, object_path): + resource = '/'.join([path, object_name]) + + minio_client = create_storage_client(bucket_prefix) minio_client.put_project_object({'test': True}, bucket, resource) assert minio_client.remove_folder_objects(bucket, path) @@ -132,8 +171,9 @@ def test_remove_existing_folder(): 'SignedHeaders=host;user-agent;x-amz-content-sha256;x-amz-date, ' 'Signature=dc971726ff2b266f208b250089b2ba0be86352efad2858145b33c2ae085e7d71' } - response = requests.get(get_url(bucket, resource), headers=headers) - assert response.status_code == 404 + resource_url = get_url(bucket_name, object_path) + response = requests.get(resource_url, headers=headers) + assert response.status_code == 404, f'Resource {resource_url} was not removed' def test_list_not_existing_folder(): @@ -143,13 +183,15 @@ def test_list_not_existing_folder(): assert minio_client.get_folder_objects('4', path) == [] -def test_list_existing_folder(): - bucket = '6' - object_name = f'{random_alphanumeric(16)}.json' - path = 'test' +@pytest.mark.parametrize('bucket_prefix, bucket, bucket_name, object_name, path, object_path', [ + (BUCKET_PREFIX, '6', f'{BUCKET_PREFIX}6', 'my_test_file_list.json', 'list', 'list/my_test_file_list.json'), + (f'test/{BUCKET_PREFIX}', '6', 'test', 'my_test_file_list.json', 'list', + f'{BUCKET_PREFIX}6/list/my_test_file_list.json'), +]) +def test_list_existing_folder(bucket_prefix, bucket, bucket_name, object_name, path, object_path): resource = '/'.join([path, object_name]) - minio_client = create_storage_client() + minio_client = create_storage_client(bucket_prefix) minio_client.put_project_object({'test': True}, bucket, resource, using_json=True) assert minio_client.get_folder_objects(bucket, path) == [resource] @@ -167,15 +209,22 @@ def test_list_dir_separators(): assert minio_client.get_folder_objects(bucket, path) == [resource] -def test_remove_project_objects(): - bucket = '8' - object_name = f'{random_alphanumeric(16)}.json' - path = 'test/' - resource = path + object_name +@pytest.mark.parametrize('bucket_prefix, bucket, bucket_name, object_name, path, object_path', [ + (BUCKET_PREFIX, '8', f'{BUCKET_PREFIX}8', 'my_test_file_objects.json', 'objects', + 'objects/my_test_file_objects.json'), + (f'test/{BUCKET_PREFIX}', '8', 'test', 'my_test_file_objects.json', 'objects', + f'{BUCKET_PREFIX}8/objects/my_test_file_objects.json'), +]) +def test_remove_project_objects(bucket_prefix, bucket, bucket_name, object_name, path, object_path): + resource = '/'.join([path, object_name]) - minio_client = create_storage_client() + minio_client = create_storage_client(bucket_prefix) minio_client.put_project_object({'test': True}, bucket, resource, using_json=True) + result = minio_client.get_project_object(bucket, resource, using_json=True) + assert isinstance(result, dict) + assert result['test'] is True + minio_client.remove_project_objects(bucket, [resource]) with pytest.raises(ValueError): minio_client.get_project_object(bucket, resource) diff --git a/test/unit/machine_learning/__init__.py b/test/unit/machine_learning/__init__.py new file mode 100644 index 00000000..2c4530c5 --- /dev/null +++ b/test/unit/machine_learning/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/unit/machine_learning/models/__init__.py b/test/unit/machine_learning/models/__init__.py new file mode 100644 index 00000000..2c4530c5 --- /dev/null +++ b/test/unit/machine_learning/models/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/unit/machine_learning/models/test_defect_type_model.py b/test/unit/machine_learning/models/test_defect_type_model.py new file mode 100644 index 00000000..91ede954 --- /dev/null +++ b/test/unit/machine_learning/models/test_defect_type_model.py @@ -0,0 +1,67 @@ +# Copyright 2024 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from app.commons import object_saving +from app.machine_learning.models.defect_type_model import DefectTypeModel +from app.utils import utils + +WEB_DRIVER_ERROR = ("org.openqa.selenium.TimeoutException: Expected condition failed: waiting for visibility of " + "element located by By.xpath: " + "//*[contains(@class,'nav-bar_menu-items') and contains(text(),'Blog')] " + "(tried for 20 second(s) with 500 milliseconds interval)") + + +@pytest.fixture(scope='session') +def object_saver() -> object_saving.ObjectSaver: + model_settings = utils.read_json_file('res', 'model_settings.json', to_json=True) + return object_saving.create_filesystem(model_settings['GLOBAL_DEFECT_TYPE_MODEL_FOLDER']) + + +@pytest.fixture(scope='session') +def defect_type_model(object_saver: object_saving.ObjectSaver) -> DefectTypeModel: + model = DefectTypeModel(object_saver) + model.load_model() + return model + + +@pytest.mark.parametrize('defect_type, expected', [ + ('nd001', 0.0), + ('pb001', 0.0), + ('ab001', 0.0), + ('si001', 1.0), + ('pd001', 0.0), + ('ab_abracadabra', 0.0), + ('pb_abracadabra', 0.0), + ('si_abracadabra', 1.0), +]) +def test_different_defect_type_predict( + defect_type_model: DefectTypeModel, defect_type: str, expected: float) -> None: + result = defect_type_model.predict([WEB_DRIVER_ERROR], defect_type) + assert result[0][0] == expected + + +@pytest.mark.parametrize('defect_type', ['ndabc', 'asdfcas', 'pb00a', 'nd_abc abc', '\n_asdcas', ' _aab']) +def test_invalid_type_error(defect_type_model: DefectTypeModel, defect_type: str) -> None: + with pytest.raises(KeyError): + defect_type_model.predict([WEB_DRIVER_ERROR], defect_type) + + +def test_load_model_again_not_loads_model(defect_type_model: DefectTypeModel): + models = defect_type_model.models + count_vectorizer_models = defect_type_model.count_vectorizer_models + defect_type_model.load_model() + assert models is defect_type_model.models + assert count_vectorizer_models is defect_type_model.count_vectorizer_models diff --git a/test/unit/utils/test_log_preparation.py b/test/unit/utils/test_log_preparation.py index 5d8f4c06..1d047354 100644 --- a/test/unit/utils/test_log_preparation.py +++ b/test/unit/utils/test_log_preparation.py @@ -22,7 +22,7 @@ def test_remove_starting_thread_name(): log = read_file_lines('test_res/test_logs', 'log_line_timestamps.txt') expected_log = read_file_lines('test_res/test_logs', 'log_line_prepared.txt') for i, line in enumerate(log): - assert log_preparation.basic_prepare(line) == expected_log[i].strip() + assert log_preparation.clean_message(log_preparation.basic_prepare(line)) == expected_log[i].strip() @pytest.mark.parametrize( @@ -37,4 +37,4 @@ def test_remove_starting_thread_name(): def test_separators_log_prepare(test_file, expected_file): log = read_file('test_res/test_logs', test_file) expected_log = read_file('test_res/test_logs', expected_file) - assert log_preparation.basic_prepare(log) == expected_log.strip() + assert log_preparation.clean_message(log_preparation.basic_prepare(log)) == expected_log.strip() diff --git a/test_res/fixtures/cluster_update_all_the_same_es_with_different_errors.json b/test_res/fixtures/cluster_update_all_the_same_es_with_different_errors.json index 7db37595..5dd50421 100644 --- a/test_res/fixtures/cluster_update_all_the_same_es_with_different_errors.json +++ b/test_res/fixtures/cluster_update_all_the_same_es_with_different_errors.json @@ -1,6 +1,6 @@ {"update":{"_index":"2","_id":4}} {"doc":{"cluster_id":"37711525315085941","cluster_message":"AssertionError error occurred\nerror found","cluster_with_numbers":true}} {"update":{"_index":"2","_id":5}} -{"doc":{"cluster_id":"48851059259117511","cluster_message":"AssertionError status code SPECIALNUMBER error occurred\nerror found","cluster_with_numbers":true}} +{"doc":{"cluster_id":"83179189436345941","cluster_message":"AssertionError status code 500 error occurred\nerror found","cluster_with_numbers":true}} {"update":{"_index":"2","_id":9}} {"doc":{"cluster_id":"90988898127574211","cluster_message":"NoSuchElementException error occurred\nerror found","cluster_with_numbers":true}} diff --git a/test_res/fixtures/cluster_update_small_logs.json b/test_res/fixtures/cluster_update_small_logs.json index e9000d36..5d885a35 100644 --- a/test_res/fixtures/cluster_update_small_logs.json +++ b/test_res/fixtures/cluster_update_small_logs.json @@ -3,10 +3,10 @@ {"update":{"_index":"2","_id":4}} {"doc":{"cluster_id":"60604459849884091","cluster_message":"error occurred twice\nAssertionError error occurred\nerror found","cluster_with_numbers":true}} {"update":{"_index":"2","_id":5}} -{"doc":{"cluster_id":"9398573272102061","cluster_message":"AssertionError status code SPECIALNUMBER error occurred","cluster_with_numbers":true}} +{"doc":{"cluster_id":"58202398056526781","cluster_message":"AssertionError status code 500 error occurred","cluster_with_numbers":true}} {"update":{"_index":"2","_id":9}} {"doc":{"cluster_id":"86465058569810291","cluster_message":"NoSuchElementException error occurred\nerror found\nassert query","cluster_with_numbers":true}} {"update":{"_index":"2","_id":"4_m"}} {"doc":{"cluster_id":"60604459849884091","cluster_message":"error occurred twice\nAssertionError error occurred\nerror found","cluster_with_numbers":true}} {"update":{"_index":"2","_id":"5_m"}} -{"doc":{"cluster_id":"9398573272102061","cluster_message":"AssertionError status code SPECIALNUMBER error occurred","cluster_with_numbers":true}} +{"doc":{"cluster_id":"58202398056526781","cluster_message":"AssertionError status code 500 error occurred","cluster_with_numbers":true}} diff --git a/test_res/fixtures/index_logs_rq.json b/test_res/fixtures/index_logs_rq.json index 38d18f83..b656edf8 100644 --- a/test_res/fixtures/index_logs_rq.json +++ b/test_res/fixtures/index_logs_rq.json @@ -1,2 +1,2 @@ {"index":{"_index":"idx2","_type":"log","_id":"1_m"}} -{"issue_type":"AB001","launch_name":"Launch 1","launch_number":0,"log_level":40000,"original_message_lines":1,"original_message_words_number":2,"message":"","test_item":1,"test_item_name":"first test","start_time":"2020-01-15 10:57:43","unique_id":"unique1","test_case_hash":-1126886180,"detected_message":"","detected_message_with_numbers":"","only_numbers":"12","merged_small_logs":"message specialnumber http localhost admin java.lang.noclassdeffounderror","stacktrace":"","urls":"http localhost admin","paths":"","message_params":"","potential_status_codes":"","found_exceptions":"java.lang.noclassdeffounderror","found_exceptions_extended":"java.lang.noclassdeffounderror lang.noclassdeffounderror noclassdeffounderror","found_tests_and_methods":"","stacktrace_extended":"","message_extended":"","detected_message_extended":"","detected_message_without_params_extended":"message specialnumber java.lang lang.noclassdeffounderror noclassdeffounderror","message_without_params_extended":"","message_without_params_and_brackets":"","detected_message_without_params_and_brackets":"","whole_message":"Message SPECIALNUMBER \n java.lang lang.NoClassDefFoundError\n","is_merged":true} +{"issue_type":"AB001","launch_name":"Launch 1","launch_number":0,"log_level":40000,"message_lines":1,"message_words_number":2,"message":"","test_item":1,"test_item_name":"first test","start_time":"2020-01-15 10:57:43","unique_id":"unique1","test_case_hash":-1126886180,"detected_message":"","detected_message_with_numbers":"","only_numbers":"12","merged_small_logs":"message specialnumber http localhost admin java.lang.noclassdeffounderror","stacktrace":"","urls":"http localhost admin","paths":"","message_params":"","potential_status_codes":"","found_exceptions":"java.lang.noclassdeffounderror","found_exceptions_extended":"java.lang.noclassdeffounderror lang.noclassdeffounderror noclassdeffounderror","found_tests_and_methods":"","stacktrace_extended":"","message_extended":"","detected_message_extended":"","detected_message_without_params_extended":"message specialnumber java.lang lang.noclassdeffounderror noclassdeffounderror","message_without_params_extended":"","message_without_params_and_brackets":"","detected_message_without_params_and_brackets":"","whole_message":"Message SPECIALNUMBER \n java.lang lang.NoClassDefFoundError\n","is_merged":true} diff --git a/test_res/fixtures/index_logs_rq_big_messages.json b/test_res/fixtures/index_logs_rq_big_messages.json index eae5fbd0..7cfb6758 100644 --- a/test_res/fixtures/index_logs_rq_big_messages.json +++ b/test_res/fixtures/index_logs_rq_big_messages.json @@ -1,4 +1,4 @@ {"index":{"_index":"2","_id":1}} -{"launch_id":1234567892,"launch_name":"Launch with test items with logs","launch_number":0,"launch_start_time":"2020-01-15 10:57:43","test_item":1,"test_item_name":"first test","unique_id":"unique1","cluster_id":"0","cluster_message":"","test_case_hash":-1126886180,"is_auto_analyzed":false,"issue_type":"ti001","log_time":"2020-01-15 10:57:43","log_level":40000,"original_message":"Message 1 \n Message 2 \n Message 3 'prod_en' /src/prod/results.html \n java.lang.NoClassDefFoundError\n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator$1@31ca8ab4 \n ca.canadiantire.steps.hybris.ws.WebserviceHybrisCustomerAndCartSteps$$EnhancerByCGLIB$$84837ae7.CGLIB$add_products_to_cart$16()","original_message_lines":5,"original_message_words_number":9,"message":"Message SPECIALNUMBER\n Message SPECIALNUMBER 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError\n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.SPECIALNUMBER","is_merged":false,"start_time":"2020-01-15 10:57:43","merged_small_logs":"","detected_message":"Message SPECIALNUMBER\n Message SPECIALNUMBER 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError\n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.SPECIALNUMBER","detected_message_with_numbers":"Message 1\n Message 2\n Message 3 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError\n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.1","stacktrace":"","only_numbers":"1 2 3","found_exceptions":"java.lang.NoClassDefFoundError","whole_message":"Message SPECIALNUMBER\nMessage SPECIALNUMBER\nMessage SPECIALNUMBER prod en src prod results html\njava lang NoClassDefFoundError\nde hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER\n","potential_status_codes":"","found_tests_and_methods":"","cluster_with_numbers":false,"urls":"","paths":"/src/prod/results.html","message_params":"prod_en","found_exceptions_extended":"java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError","detected_message_extended":"Message 1\n Message 2\n Message 3 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError \n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.1","detected_message_without_params_extended":"Message SPECIALNUMBER\nMessage SPECIALNUMBER prod en src prod results html\njava lang NoClassDefFoundError\nde hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER","stacktrace_extended":"","message_extended":"Message SPECIALNUMBER\n Message SPECIALNUMBER 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError \n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.SPECIALNUMBER MandatoryAttributesValidator.SPECIALNUMBER SPECIALNUMBER ","message_without_params_extended":"Message SPECIALNUMBER\n Message SPECIALNUMBER prod en src prod results html\n java lang NoClassDefFoundError\n de hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER","detected_message_without_params_and_brackets":"Message SPECIALNUMBER\nMessage SPECIALNUMBER prod en src prod results html\njava lang NoClassDefFoundError\nde hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER","message_without_params_and_brackets":"Message SPECIALNUMBER\n Message SPECIALNUMBER prod en src prod results html\n java lang NoClassDefFoundError\n de hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER"} +{"launch_id":1234567892,"launch_name":"Launch with test items with logs","launch_number":0,"launch_start_time":"2020-01-15 10:57:43","test_item":1,"test_item_name":"first test","unique_id":"unique1","cluster_id":"0","cluster_message":"","test_case_hash":-1126886180,"is_auto_analyzed":false,"issue_type":"ti001","log_time":"2020-01-15 10:57:43","log_level":40000,"original_message":"Message 1 \n Message 2 \n Message 3 'prod_en' /src/prod/results.html \n java.lang.NoClassDefFoundError\n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator$1@31ca8ab4 \n ca.canadiantire.steps.hybris.ws.WebserviceHybrisCustomerAndCartSteps$$EnhancerByCGLIB$$84837ae7.CGLIB$add_products_to_cart$16()","message_lines":5,"message_words_number":9,"message":"Message SPECIALNUMBER\n Message SPECIALNUMBER 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError\n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.SPECIALNUMBER","is_merged":false,"start_time":"2020-01-15 10:57:43","merged_small_logs":"","detected_message":"Message SPECIALNUMBER\n Message SPECIALNUMBER 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError\n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.SPECIALNUMBER","detected_message_with_numbers":"Message 1\n Message 2\n Message 3 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError\n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.1","stacktrace":"","only_numbers":"1 2 3","found_exceptions":"java.lang.NoClassDefFoundError","whole_message":"Message SPECIALNUMBER\nMessage SPECIALNUMBER\nMessage SPECIALNUMBER prod en src prod results html\njava lang NoClassDefFoundError\nde hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER\n","potential_status_codes":"","found_tests_and_methods":"","cluster_with_numbers":false,"urls":"","paths":"/src/prod/results.html","message_params":"prod_en","found_exceptions_extended":"java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError","detected_message_extended":"Message 1\n Message 2\n Message 3 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError \n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.1","detected_message_without_params_extended":"Message SPECIALNUMBER\nMessage SPECIALNUMBER prod en src prod results html\njava lang NoClassDefFoundError\nde hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER","stacktrace_extended":"","message_extended":"Message SPECIALNUMBER\n Message SPECIALNUMBER 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError \n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.SPECIALNUMBER MandatoryAttributesValidator.SPECIALNUMBER SPECIALNUMBER ","message_without_params_extended":"Message SPECIALNUMBER\n Message SPECIALNUMBER prod en src prod results html\n java lang NoClassDefFoundError\n de hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER","detected_message_without_params_and_brackets":"Message SPECIALNUMBER\nMessage SPECIALNUMBER prod en src prod results html\njava lang NoClassDefFoundError\nde hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER","message_without_params_and_brackets":"Message SPECIALNUMBER\n Message SPECIALNUMBER prod en src prod results html\n java lang NoClassDefFoundError\n de hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER"} {"index":{"_index":"2","_id":2}} -{"launch_id":1234567892,"launch_name":"Launch with test items with logs","launch_number":0,"launch_start_time":"2020-01-15 10:57:43","test_item":1,"test_item_name":"first test","unique_id":"unique1","cluster_id":"0","cluster_message":"","test_case_hash":-1126886180,"is_auto_analyzed":false,"issue_type":"ti001","log_time":"2020-01-15 10:57:43","log_level":40000,"original_message":"Message 2 \n Message 4 \n Message 5 http:localhost/admin \n java.lang.NoClassDefFoundError\n For documentation on this error, please visit https://www.seleniumhq.org/exceptions/stale_element_reference.html","original_message_lines":4,"original_message_words_number":7,"message":"Message SPECIALNUMBER\n Message SPECIALNUMBER http : localhost/admin\n java.lang.NoClassDefFoundError","is_merged":false,"start_time":"2020-01-15 10:57:43","merged_small_logs":"","detected_message":"Message SPECIALNUMBER\n Message SPECIALNUMBER http : localhost/admin\n java.lang.NoClassDefFoundError","detected_message_with_numbers":"Message 2\n Message 4\n Message 5 http : localhost/admin\n java.lang.NoClassDefFoundError","stacktrace":"","only_numbers":"2 4 5","found_exceptions":"java.lang.NoClassDefFoundError","whole_message":"Message SPECIALNUMBER\nMessage SPECIALNUMBER\nMessage SPECIALNUMBER http localhost admin\njava lang NoClassDefFoundError\n","potential_status_codes":"","found_tests_and_methods":"","cluster_with_numbers":false,"urls":"","paths":"","message_params":"","found_exceptions_extended":"java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError","detected_message_extended":"Message 2\n Message 4\n Message 5 http : localhost/admin\n java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError ","detected_message_without_params_extended":"Message SPECIALNUMBER\nMessage SPECIALNUMBER http localhost admin\njava lang NoClassDefFoundError","stacktrace_extended":"","message_extended":"Message SPECIALNUMBER\n Message SPECIALNUMBER http : localhost/admin\n java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError ","message_without_params_extended":"Message SPECIALNUMBER\n Message SPECIALNUMBER http localhost admin\n java lang NoClassDefFoundError","detected_message_without_params_and_brackets":"Message SPECIALNUMBER\nMessage SPECIALNUMBER http localhost admin\njava lang NoClassDefFoundError","message_without_params_and_brackets":"Message SPECIALNUMBER\n Message SPECIALNUMBER http localhost admin\n java lang NoClassDefFoundError"} +{"launch_id":1234567892,"launch_name":"Launch with test items with logs","launch_number":0,"launch_start_time":"2020-01-15 10:57:43","test_item":1,"test_item_name":"first test","unique_id":"unique1","cluster_id":"0","cluster_message":"","test_case_hash":-1126886180,"is_auto_analyzed":false,"issue_type":"ti001","log_time":"2020-01-15 10:57:43","log_level":40000,"original_message":"Message 2 \n Message 4 \n Message 5 http:localhost/admin \n java.lang.NoClassDefFoundError\n For documentation on this error, please visit https://www.seleniumhq.org/exceptions/stale_element_reference.html","message_lines":4,"message_words_number":7,"message":"Message SPECIALNUMBER\n Message SPECIALNUMBER http : localhost/admin\n java.lang.NoClassDefFoundError","is_merged":false,"start_time":"2020-01-15 10:57:43","merged_small_logs":"","detected_message":"Message SPECIALNUMBER\n Message SPECIALNUMBER http : localhost/admin\n java.lang.NoClassDefFoundError","detected_message_with_numbers":"Message 2\n Message 4\n Message 5 http : localhost/admin\n java.lang.NoClassDefFoundError","stacktrace":"","only_numbers":"2 4 5","found_exceptions":"java.lang.NoClassDefFoundError","whole_message":"Message SPECIALNUMBER\nMessage SPECIALNUMBER\nMessage SPECIALNUMBER http localhost admin\njava lang NoClassDefFoundError\n","potential_status_codes":"","found_tests_and_methods":"","cluster_with_numbers":false,"urls":"","paths":"","message_params":"","found_exceptions_extended":"java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError","detected_message_extended":"Message 2\n Message 4\n Message 5 http : localhost/admin\n java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError ","detected_message_without_params_extended":"Message SPECIALNUMBER\nMessage SPECIALNUMBER http localhost admin\njava lang NoClassDefFoundError","stacktrace_extended":"","message_extended":"Message SPECIALNUMBER\n Message SPECIALNUMBER http : localhost/admin\n java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError ","message_without_params_extended":"Message SPECIALNUMBER\n Message SPECIALNUMBER http localhost admin\n java lang NoClassDefFoundError","detected_message_without_params_and_brackets":"Message SPECIALNUMBER\nMessage SPECIALNUMBER http localhost admin\njava lang NoClassDefFoundError","message_without_params_and_brackets":"Message SPECIALNUMBER\n Message SPECIALNUMBER http localhost admin\n java lang NoClassDefFoundError"} diff --git a/test_res/fixtures/index_logs_rq_big_messages_with_clusters.json b/test_res/fixtures/index_logs_rq_big_messages_with_clusters.json index 8c6c5f09..905189c2 100644 --- a/test_res/fixtures/index_logs_rq_big_messages_with_clusters.json +++ b/test_res/fixtures/index_logs_rq_big_messages_with_clusters.json @@ -1,4 +1,4 @@ {"index":{"_index":"2","_id":1}} -{"launch_id":1234567892,"launch_name":"Launch with test items with logs","launch_number":0,"launch_start_time":"2020-01-15 10:57:43","test_item":1,"test_item_name":"first test","unique_id":"unique1","cluster_id":"2727777272727727721","cluster_message":"","test_case_hash":-1126886180,"is_auto_analyzed":false,"issue_type":"ti001","log_time":"2020-01-15 10:57:43","log_level":40000,"original_message":"Message 1 \n Message 2 \n Message 3 'prod_en' /src/prod/results.html \n java.lang.NoClassDefFoundError\n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator$1@31ca8ab4 \n ca.canadiantire.steps.hybris.ws.WebserviceHybrisCustomerAndCartSteps$$EnhancerByCGLIB$$84837ae7.CGLIB$add_products_to_cart$16()","original_message_lines":5,"original_message_words_number":9,"message":"Message SPECIALNUMBER\n Message SPECIALNUMBER 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError\n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.SPECIALNUMBER","is_merged":false,"start_time":"2020-01-15 10:57:43","merged_small_logs":"","detected_message":"Message SPECIALNUMBER\n Message SPECIALNUMBER 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError\n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.SPECIALNUMBER","detected_message_with_numbers":"Message 1\n Message 2\n Message 3 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError\n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.1","stacktrace":"","only_numbers":"1 2 3","found_exceptions":"java.lang.NoClassDefFoundError","whole_message":"Message SPECIALNUMBER\nMessage SPECIALNUMBER\nMessage SPECIALNUMBER prod en src prod results html\njava lang NoClassDefFoundError\nde hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER\n","potential_status_codes":"","found_tests_and_methods":"","cluster_with_numbers":true,"urls":"","paths":"/src/prod/results.html","message_params":"prod_en","found_exceptions_extended":"java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError","detected_message_extended":"Message 1\n Message 2\n Message 3 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError \n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.1","detected_message_without_params_extended":"Message SPECIALNUMBER\nMessage SPECIALNUMBER prod en src prod results html\njava lang NoClassDefFoundError\nde hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER","stacktrace_extended":"","message_extended":"Message SPECIALNUMBER\n Message SPECIALNUMBER 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError \n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.SPECIALNUMBER MandatoryAttributesValidator.SPECIALNUMBER SPECIALNUMBER ","message_without_params_extended":"Message SPECIALNUMBER\n Message SPECIALNUMBER prod en src prod results html\n java lang NoClassDefFoundError\n de hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER","detected_message_without_params_and_brackets":"Message SPECIALNUMBER\nMessage SPECIALNUMBER prod en src prod results html\njava lang NoClassDefFoundError\nde hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER","message_without_params_and_brackets":"Message SPECIALNUMBER\n Message SPECIALNUMBER prod en src prod results html\n java lang NoClassDefFoundError\n de hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER"} +{"launch_id":1234567892,"launch_name":"Launch with test items with logs","launch_number":0,"launch_start_time":"2020-01-15 10:57:43","test_item":1,"test_item_name":"first test","unique_id":"unique1","cluster_id":"2727777272727727721","cluster_message":"","test_case_hash":-1126886180,"is_auto_analyzed":false,"issue_type":"ti001","log_time":"2020-01-15 10:57:43","log_level":40000,"original_message":"Message 1 \n Message 2 \n Message 3 'prod_en' /src/prod/results.html \n java.lang.NoClassDefFoundError\n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator$1@31ca8ab4 \n ca.canadiantire.steps.hybris.ws.WebserviceHybrisCustomerAndCartSteps$$EnhancerByCGLIB$$84837ae7.CGLIB$add_products_to_cart$16()","message_lines":5,"message_words_number":9,"message":"Message SPECIALNUMBER\n Message SPECIALNUMBER 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError\n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.SPECIALNUMBER","is_merged":false,"start_time":"2020-01-15 10:57:43","merged_small_logs":"","detected_message":"Message SPECIALNUMBER\n Message SPECIALNUMBER 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError\n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.SPECIALNUMBER","detected_message_with_numbers":"Message 1\n Message 2\n Message 3 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError\n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.1","stacktrace":"","only_numbers":"1 2 3","found_exceptions":"java.lang.NoClassDefFoundError","whole_message":"Message SPECIALNUMBER\nMessage SPECIALNUMBER\nMessage SPECIALNUMBER prod en src prod results html\njava lang NoClassDefFoundError\nde hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER\n","potential_status_codes":"","found_tests_and_methods":"","cluster_with_numbers":true,"urls":"","paths":"/src/prod/results.html","message_params":"prod_en","found_exceptions_extended":"java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError","detected_message_extended":"Message 1\n Message 2\n Message 3 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError \n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.1","detected_message_without_params_extended":"Message SPECIALNUMBER\nMessage SPECIALNUMBER prod en src prod results html\njava lang NoClassDefFoundError\nde hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER","stacktrace_extended":"","message_extended":"Message SPECIALNUMBER\n Message SPECIALNUMBER 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError \n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.SPECIALNUMBER MandatoryAttributesValidator.SPECIALNUMBER SPECIALNUMBER ","message_without_params_extended":"Message SPECIALNUMBER\n Message SPECIALNUMBER prod en src prod results html\n java lang NoClassDefFoundError\n de hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER","detected_message_without_params_and_brackets":"Message SPECIALNUMBER\nMessage SPECIALNUMBER prod en src prod results html\njava lang NoClassDefFoundError\nde hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER","message_without_params_and_brackets":"Message SPECIALNUMBER\n Message SPECIALNUMBER prod en src prod results html\n java lang NoClassDefFoundError\n de hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER"} {"index":{"_index":"2","_id":2}} -{"launch_id":1234567892,"launch_name":"Launch with test items with logs","launch_number":0,"launch_start_time":"2020-01-15 10:57:43","test_item":1,"test_item_name":"first test","unique_id":"unique1","cluster_id":"0","cluster_message":"","test_case_hash":-1126886180,"is_auto_analyzed":false,"issue_type":"ti001","log_time":"2020-01-15 10:57:43","log_level":40000,"original_message":"Message 2 \n Message 4 \n Message 5 http:localhost/admin \n java.lang.NoClassDefFoundError\n For documentation on this error, please visit https://www.seleniumhq.org/exceptions/stale_element_reference.html","original_message_lines":4,"original_message_words_number":7,"message":"Message SPECIALNUMBER\n Message SPECIALNUMBER http : localhost/admin\n java.lang.NoClassDefFoundError","is_merged":false,"start_time":"2020-01-15 10:57:43","merged_small_logs":"","detected_message":"Message SPECIALNUMBER\n Message SPECIALNUMBER http : localhost/admin\n java.lang.NoClassDefFoundError","detected_message_with_numbers":"Message 2\n Message 4\n Message 5 http : localhost/admin\n java.lang.NoClassDefFoundError","stacktrace":"","only_numbers":"2 4 5","found_exceptions":"java.lang.NoClassDefFoundError","whole_message":"Message SPECIALNUMBER\nMessage SPECIALNUMBER\nMessage SPECIALNUMBER http localhost admin\njava lang NoClassDefFoundError\n","potential_status_codes":"","found_tests_and_methods":"","cluster_with_numbers":false,"urls":"","paths":"","message_params":"","found_exceptions_extended":"java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError","detected_message_extended":"Message 2\n Message 4\n Message 5 http : localhost/admin\n java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError ","detected_message_without_params_extended":"Message SPECIALNUMBER\nMessage SPECIALNUMBER http localhost admin\njava lang NoClassDefFoundError","stacktrace_extended":"","message_extended":"Message SPECIALNUMBER\n Message SPECIALNUMBER http : localhost/admin\n java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError ","message_without_params_extended":"Message SPECIALNUMBER\n Message SPECIALNUMBER http localhost admin\n java lang NoClassDefFoundError","detected_message_without_params_and_brackets":"Message SPECIALNUMBER\nMessage SPECIALNUMBER http localhost admin\njava lang NoClassDefFoundError","message_without_params_and_brackets":"Message SPECIALNUMBER\n Message SPECIALNUMBER http localhost admin\n java lang NoClassDefFoundError"} +{"launch_id":1234567892,"launch_name":"Launch with test items with logs","launch_number":0,"launch_start_time":"2020-01-15 10:57:43","test_item":1,"test_item_name":"first test","unique_id":"unique1","cluster_id":"0","cluster_message":"","test_case_hash":-1126886180,"is_auto_analyzed":false,"issue_type":"ti001","log_time":"2020-01-15 10:57:43","log_level":40000,"original_message":"Message 2 \n Message 4 \n Message 5 http:localhost/admin \n java.lang.NoClassDefFoundError\n For documentation on this error, please visit https://www.seleniumhq.org/exceptions/stale_element_reference.html","message_lines":4,"message_words_number":7,"message":"Message SPECIALNUMBER\n Message SPECIALNUMBER http : localhost/admin\n java.lang.NoClassDefFoundError","is_merged":false,"start_time":"2020-01-15 10:57:43","merged_small_logs":"","detected_message":"Message SPECIALNUMBER\n Message SPECIALNUMBER http : localhost/admin\n java.lang.NoClassDefFoundError","detected_message_with_numbers":"Message 2\n Message 4\n Message 5 http : localhost/admin\n java.lang.NoClassDefFoundError","stacktrace":"","only_numbers":"2 4 5","found_exceptions":"java.lang.NoClassDefFoundError","whole_message":"Message SPECIALNUMBER\nMessage SPECIALNUMBER\nMessage SPECIALNUMBER http localhost admin\njava lang NoClassDefFoundError\n","potential_status_codes":"","found_tests_and_methods":"","cluster_with_numbers":false,"urls":"","paths":"","message_params":"","found_exceptions_extended":"java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError","detected_message_extended":"Message 2\n Message 4\n Message 5 http : localhost/admin\n java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError ","detected_message_without_params_extended":"Message SPECIALNUMBER\nMessage SPECIALNUMBER http localhost admin\njava lang NoClassDefFoundError","stacktrace_extended":"","message_extended":"Message SPECIALNUMBER\n Message SPECIALNUMBER http : localhost/admin\n java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError ","message_without_params_extended":"Message SPECIALNUMBER\n Message SPECIALNUMBER http localhost admin\n java lang NoClassDefFoundError","detected_message_without_params_and_brackets":"Message SPECIALNUMBER\nMessage SPECIALNUMBER http localhost admin\njava lang NoClassDefFoundError","message_without_params_and_brackets":"Message SPECIALNUMBER\n Message SPECIALNUMBER http localhost admin\n java lang NoClassDefFoundError"} diff --git a/test_res/fixtures/index_logs_rq_different_log_level.json b/test_res/fixtures/index_logs_rq_different_log_level.json index 688ee101..a39c9673 100644 --- a/test_res/fixtures/index_logs_rq_different_log_level.json +++ b/test_res/fixtures/index_logs_rq_different_log_level.json @@ -1,2 +1,2 @@ {"index":{"_index":"2","_id":1}} -{"launch_id":1234567892,"launch_name":"Launch with test items with logs","launch_number":0,"launch_start_time":"2020-01-15 10:57:43","test_item":1,"test_item_name":"first test","unique_id":"unique1","cluster_id":"0","cluster_message":"","test_case_hash":-1126886180,"is_auto_analyzed":false,"issue_type":"ti001","log_time":"2020-01-15 10:57:43","log_level":40000,"original_message":"Message 1 \n java.lang.reflect.Method.invoke(Method.java:498) \n message error caused by exception\n ... 34 more","original_message_lines":3,"original_message_words_number":8,"message":" Message SPECIALNUMBER \n java.lang.reflect.Method.invoke(Method.java : SPECIALNUMBER)\n message error caused by exception","is_merged":false,"start_time":"2020-01-15 10:57:43","merged_small_logs":"","detected_message":" Message SPECIALNUMBER \n message error caused by exception","detected_message_with_numbers":" Message 1 \n message error caused by exception","stacktrace":" java.lang.reflect.Method.invoke","only_numbers":"1","found_exceptions":"","whole_message":" Message SPECIALNUMBER\nmessage error caused by exception\n java.lang.reflect.Method.invoke","potential_status_codes":"","found_tests_and_methods":"","cluster_with_numbers":false,"urls":"","paths":"","message_params":"","found_exceptions_extended":"","detected_message_extended":" Message 1 \n message error caused by exception","detected_message_without_params_extended":" Message SPECIALNUMBER\nmessage error caused by exception","stacktrace_extended":" java.lang.reflect.Method.invoke Method.invoke invoke ","message_extended":" Message SPECIALNUMBER \n java.lang.reflect.Method.invoke Method.invoke invoke (Method.java : SPECIALNUMBER)\n message error caused by exception","message_without_params_extended":" Message SPECIALNUMBER \n java lang reflect Method invoke Method java SPECIALNUMBER \n message error caused by exception","detected_message_without_params_and_brackets":" Message SPECIALNUMBER\nmessage error caused by exception","message_without_params_and_brackets":" Message SPECIALNUMBER \n java lang reflect Method invoke Method java SPECIALNUMBER \n message error caused by exception"} +{"launch_id":1234567892,"launch_name":"Launch with test items with logs","launch_number":0,"launch_start_time":"2020-01-15 10:57:43","test_item":1,"test_item_name":"first test","unique_id":"unique1","cluster_id":"0","cluster_message":"","test_case_hash":-1126886180,"is_auto_analyzed":false,"issue_type":"ti001","log_time":"2020-01-15 10:57:43","log_level":40000,"original_message":"Message 1 \n java.lang.reflect.Method.invoke(Method.java:498) \n message error caused by exception\n ... 34 more","message_lines":3,"message_words_number":8,"message":" Message SPECIALNUMBER \n java.lang.reflect.Method.invoke(Method.java : SPECIALNUMBER)\n message error caused by exception","is_merged":false,"start_time":"2020-01-15 10:57:43","merged_small_logs":"","detected_message":" Message SPECIALNUMBER \n message error caused by exception","detected_message_with_numbers":" Message 1 \n message error caused by exception","stacktrace":" java.lang.reflect.Method.invoke","only_numbers":"1","found_exceptions":"","whole_message":" Message SPECIALNUMBER\nmessage error caused by exception\n java.lang.reflect.Method.invoke","potential_status_codes":"","found_tests_and_methods":"","cluster_with_numbers":false,"urls":"","paths":"","message_params":"","found_exceptions_extended":"","detected_message_extended":" Message 1 \n message error caused by exception","detected_message_without_params_extended":" Message SPECIALNUMBER\nmessage error caused by exception","stacktrace_extended":" java.lang.reflect.Method.invoke Method.invoke invoke ","message_extended":" Message SPECIALNUMBER \n java.lang.reflect.Method.invoke Method.invoke invoke (Method.java : SPECIALNUMBER)\n message error caused by exception","message_without_params_extended":" Message SPECIALNUMBER \n java lang reflect Method invoke Method java SPECIALNUMBER \n message error caused by exception","detected_message_without_params_and_brackets":" Message SPECIALNUMBER\nmessage error caused by exception","message_without_params_and_brackets":" Message SPECIALNUMBER \n java lang reflect Method invoke Method java SPECIALNUMBER \n message error caused by exception"} diff --git a/test_res/fixtures/index_logs_rq_different_log_level_merged.json b/test_res/fixtures/index_logs_rq_different_log_level_merged.json index 38d18f83..b656edf8 100644 --- a/test_res/fixtures/index_logs_rq_different_log_level_merged.json +++ b/test_res/fixtures/index_logs_rq_different_log_level_merged.json @@ -1,2 +1,2 @@ {"index":{"_index":"idx2","_type":"log","_id":"1_m"}} -{"issue_type":"AB001","launch_name":"Launch 1","launch_number":0,"log_level":40000,"original_message_lines":1,"original_message_words_number":2,"message":"","test_item":1,"test_item_name":"first test","start_time":"2020-01-15 10:57:43","unique_id":"unique1","test_case_hash":-1126886180,"detected_message":"","detected_message_with_numbers":"","only_numbers":"12","merged_small_logs":"message specialnumber http localhost admin java.lang.noclassdeffounderror","stacktrace":"","urls":"http localhost admin","paths":"","message_params":"","potential_status_codes":"","found_exceptions":"java.lang.noclassdeffounderror","found_exceptions_extended":"java.lang.noclassdeffounderror lang.noclassdeffounderror noclassdeffounderror","found_tests_and_methods":"","stacktrace_extended":"","message_extended":"","detected_message_extended":"","detected_message_without_params_extended":"message specialnumber java.lang lang.noclassdeffounderror noclassdeffounderror","message_without_params_extended":"","message_without_params_and_brackets":"","detected_message_without_params_and_brackets":"","whole_message":"Message SPECIALNUMBER \n java.lang lang.NoClassDefFoundError\n","is_merged":true} +{"issue_type":"AB001","launch_name":"Launch 1","launch_number":0,"log_level":40000,"message_lines":1,"message_words_number":2,"message":"","test_item":1,"test_item_name":"first test","start_time":"2020-01-15 10:57:43","unique_id":"unique1","test_case_hash":-1126886180,"detected_message":"","detected_message_with_numbers":"","only_numbers":"12","merged_small_logs":"message specialnumber http localhost admin java.lang.noclassdeffounderror","stacktrace":"","urls":"http localhost admin","paths":"","message_params":"","potential_status_codes":"","found_exceptions":"java.lang.noclassdeffounderror","found_exceptions_extended":"java.lang.noclassdeffounderror lang.noclassdeffounderror noclassdeffounderror","found_tests_and_methods":"","stacktrace_extended":"","message_extended":"","detected_message_extended":"","detected_message_without_params_extended":"message specialnumber java.lang lang.noclassdeffounderror noclassdeffounderror","message_without_params_extended":"","message_without_params_and_brackets":"","detected_message_without_params_and_brackets":"","whole_message":"Message SPECIALNUMBER \n java.lang lang.NoClassDefFoundError\n","is_merged":true} diff --git a/test_res/fixtures/index_logs_rq_different_log_level_with_prefix.json b/test_res/fixtures/index_logs_rq_different_log_level_with_prefix.json index 56e9ac11..4c7e32cc 100644 --- a/test_res/fixtures/index_logs_rq_different_log_level_with_prefix.json +++ b/test_res/fixtures/index_logs_rq_different_log_level_with_prefix.json @@ -1,2 +1,2 @@ {"index":{"_index":"rp_2","_id":1}} -{"launch_id":1234567892,"launch_name":"Launch with test items with logs","launch_number":0,"launch_start_time":"2020-01-15 10:57:43","test_item":1,"test_item_name":"first test","unique_id":"unique1","cluster_id":"0","cluster_message":"","test_case_hash":-1126886180,"is_auto_analyzed":false,"issue_type":"ti001","log_time":"2020-01-15 10:57:43","log_level":40000,"original_message":"Message 1 \n java.lang.reflect.Method.invoke(Method.java:498) \n message error caused by exception\n ... 34 more","original_message_lines":3,"original_message_words_number":8,"message":" Message SPECIALNUMBER \n java.lang.reflect.Method.invoke(Method.java : SPECIALNUMBER)\n message error caused by exception","is_merged":false,"start_time":"2020-01-15 10:57:43","merged_small_logs":"","detected_message":" Message SPECIALNUMBER \n message error caused by exception","detected_message_with_numbers":" Message 1 \n message error caused by exception","stacktrace":" java.lang.reflect.Method.invoke","only_numbers":"1","found_exceptions":"","whole_message":" Message SPECIALNUMBER\nmessage error caused by exception\n java.lang.reflect.Method.invoke","potential_status_codes":"","found_tests_and_methods":"","cluster_with_numbers":false,"urls":"","paths":"","message_params":"","found_exceptions_extended":"","detected_message_extended":" Message 1 \n message error caused by exception","detected_message_without_params_extended":" Message SPECIALNUMBER\nmessage error caused by exception","stacktrace_extended":" java.lang.reflect.Method.invoke Method.invoke invoke ","message_extended":" Message SPECIALNUMBER \n java.lang.reflect.Method.invoke Method.invoke invoke (Method.java : SPECIALNUMBER)\n message error caused by exception","message_without_params_extended":" Message SPECIALNUMBER \n java lang reflect Method invoke Method java SPECIALNUMBER \n message error caused by exception","detected_message_without_params_and_brackets":" Message SPECIALNUMBER\nmessage error caused by exception","message_without_params_and_brackets":" Message SPECIALNUMBER \n java lang reflect Method invoke Method java SPECIALNUMBER \n message error caused by exception"} +{"launch_id":1234567892,"launch_name":"Launch with test items with logs","launch_number":0,"launch_start_time":"2020-01-15 10:57:43","test_item":1,"test_item_name":"first test","unique_id":"unique1","cluster_id":"0","cluster_message":"","test_case_hash":-1126886180,"is_auto_analyzed":false,"issue_type":"ti001","log_time":"2020-01-15 10:57:43","log_level":40000,"original_message":"Message 1 \n java.lang.reflect.Method.invoke(Method.java:498) \n message error caused by exception\n ... 34 more","message_lines":3,"message_words_number":8,"message":" Message SPECIALNUMBER \n java.lang.reflect.Method.invoke(Method.java : SPECIALNUMBER)\n message error caused by exception","is_merged":false,"start_time":"2020-01-15 10:57:43","merged_small_logs":"","detected_message":" Message SPECIALNUMBER \n message error caused by exception","detected_message_with_numbers":" Message 1 \n message error caused by exception","stacktrace":" java.lang.reflect.Method.invoke","only_numbers":"1","found_exceptions":"","whole_message":" Message SPECIALNUMBER\nmessage error caused by exception\n java.lang.reflect.Method.invoke","potential_status_codes":"","found_tests_and_methods":"","cluster_with_numbers":false,"urls":"","paths":"","message_params":"","found_exceptions_extended":"","detected_message_extended":" Message 1 \n message error caused by exception","detected_message_without_params_extended":" Message SPECIALNUMBER\nmessage error caused by exception","stacktrace_extended":" java.lang.reflect.Method.invoke Method.invoke invoke ","message_extended":" Message SPECIALNUMBER \n java.lang.reflect.Method.invoke Method.invoke invoke (Method.java : SPECIALNUMBER)\n message error caused by exception","message_without_params_extended":" Message SPECIALNUMBER \n java lang reflect Method invoke Method java SPECIALNUMBER \n message error caused by exception","detected_message_without_params_and_brackets":" Message SPECIALNUMBER\nmessage error caused by exception","message_without_params_and_brackets":" Message SPECIALNUMBER \n java lang reflect Method invoke Method java SPECIALNUMBER \n message error caused by exception"} diff --git a/test_res/fixtures/one_hit_search_rs.json b/test_res/fixtures/one_hit_search_rs.json index e5ff0a72..9e9024d3 100644 --- a/test_res/fixtures/one_hit_search_rs.json +++ b/test_res/fixtures/one_hit_search_rs.json @@ -24,8 +24,8 @@ "launch_name": "Launch 1", "launch_number": 0, "log_level": 40000, - "original_message_lines": 1, - "original_message_words_number": 2, + "message_lines": 1, + "message_words_number": 2, "message": "Message SPECIALNUMBER http : localhost/admin \n java.lang.NoClassDefFoundError\r", "test_item": 1, "test_item_name": "first test", diff --git a/test_res/fixtures/one_hit_search_rs_clustering.json b/test_res/fixtures/one_hit_search_rs_clustering.json index d39b37fb..63a622da 100644 --- a/test_res/fixtures/one_hit_search_rs_clustering.json +++ b/test_res/fixtures/one_hit_search_rs_clustering.json @@ -24,8 +24,8 @@ "launch_id": 1, "launch_name": "Launch 1", "log_level": 40000, - "original_message_lines": 1, - "original_message_words_number": 2, + "message_lines": 1, + "message_words_number": 2, "message": "error occurred \n error found \n error mined", "whole_message": "error occurred \n error found \n error mined", "test_item": 12, diff --git a/test_res/fixtures/one_hit_search_rs_explained.json b/test_res/fixtures/one_hit_search_rs_explained.json index b81ae096..cf257523 100644 --- a/test_res/fixtures/one_hit_search_rs_explained.json +++ b/test_res/fixtures/one_hit_search_rs_explained.json @@ -24,8 +24,8 @@ "launch_name": "Launch 1", "launch_id": 13, "log_level": 40000, - "original_message_lines": 28, - "original_message_words_number": 50, + "message_lines": 28, + "message_words_number": 50, "message": "java.lang.NullPointerException status 200\n\tat tv.horizon.web.ta.ui.elements.assets.Promotion.setFeed(Promotion.java:)\n\tat tv.horizon.web.ta.ui.elements.assets.Promotion.(Promotion.java:)\n\tat tv.horizon.web.ta.services.navigation.AssetsManager.checkTitleCardForPromotions(AssetsManager.java:)\n\tat tv.horizon.web.tests.checklist.AspotAndPromotionChecklist.checkPromotionFeeds(AspotAndPromotionChecklist.java:)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:)\n\tat java.lang.reflect.Method.invoke(Method.java:)\n\tat org.testng.internal.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:)\n\tat org.testng.internal.MethodInvocationHelper$.runTestMethod(MethodInvocationHelper.java:)\n\tat org.springframework.test.context.testng.AbstractTestNGSpringContextTests.run(AbstractTestNGSpringContextTests.java:)\n\tat org.testng.internal.MethodInvocationHelper.invokeHookable(MethodInvocationHelper.java:)\n\tat org.testng.internal.Invoker.invokeMethod(Invoker.java:)\n\tat org.testng.internal.Invoker.invokeTestMethod(Invoker.java:)\n\tat org.testng.internal.Invoker.invokeTestMethods(Invoker.java:)\n\tat org.testng.internal.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:)\n\tat org.testng.internal.TestMethodWorker.run(TestMethodWorker.java:)\n\tat org.testng.TestRunner.privateRun(TestRunner.java:)\n\tat org.testng.TestRunner.run(TestRunner.java:)\n\tat org.testng.SuiteRunner.runTest(SuiteRunner.java:)\n\tat org.testng.SuiteRunner.access$(SuiteRunner.java:)\n\tat org.testng.SuiteRunner$SuiteWorker.run(SuiteRunner.java:)\n\tat org.testng.internal.thread.ThreadUtil$.call(ThreadUtil.java:)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:)\n\tat java.lang.Thread.run(Thread.java:)\n", "test_item": 1, "test_item_name": "first test", diff --git a/test_res/fixtures/one_hit_search_rs_explained_wo_params.json b/test_res/fixtures/one_hit_search_rs_explained_wo_params.json index f86f6f9e..baec4476 100644 --- a/test_res/fixtures/one_hit_search_rs_explained_wo_params.json +++ b/test_res/fixtures/one_hit_search_rs_explained_wo_params.json @@ -22,8 +22,8 @@ "issue_type": "AB001", "launch_name": "Launch 1", "log_level": 40000, - "original_message_lines": 28, - "original_message_words_number": 50, + "message_lines": 28, + "message_words_number": 50, "message": "java.lang.NullPointerException 'sdf qwerty'\n\tat tv.horizon.web.ta.ui.elements.assets.Promotion.setFeed(Promotion.java:)\n\tat tv.horizon.web.ta.ui.elements.assets.Promotion.(Promotion.java:)\n\tat tv.horizon.web.ta.services.navigation.AssetsManager.checkTitleCardForPromotions(AssetsManager.java:)\n\tat tv.horizon.web.tests.checklist.AspotAndPromotionChecklist.checkPromotionFeeds(AspotAndPromotionChecklist.java:)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:)\n\tat java.lang.reflect.Method.invoke(Method.java:)\n\tat org.testng.internal.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:)\n\tat org.testng.internal.MethodInvocationHelper$.runTestMethod(MethodInvocationHelper.java:)\n\tat org.springframework.test.context.testng.AbstractTestNGSpringContextTests.run(AbstractTestNGSpringContextTests.java:)\n\tat org.testng.internal.MethodInvocationHelper.invokeHookable(MethodInvocationHelper.java:)\n\tat org.testng.internal.Invoker.invokeMethod(Invoker.java:)\n\tat org.testng.internal.Invoker.invokeTestMethod(Invoker.java:)\n\tat org.testng.internal.Invoker.invokeTestMethods(Invoker.java:)\n\tat org.testng.internal.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:)\n\tat org.testng.internal.TestMethodWorker.run(TestMethodWorker.java:)\n\tat org.testng.TestRunner.privateRun(TestRunner.java:)\n\tat org.testng.TestRunner.run(TestRunner.java:)\n\tat org.testng.SuiteRunner.runTest(SuiteRunner.java:)\n\tat org.testng.SuiteRunner.access$(SuiteRunner.java:)\n\tat org.testng.SuiteRunner$SuiteWorker.run(SuiteRunner.java:)\n\tat org.testng.internal.thread.ThreadUtil$.call(ThreadUtil.java:)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:)\n\tat java.lang.Thread.run(Thread.java:)\n", "test_item": 3, "unique_id": "unique1", diff --git a/test_res/fixtures/one_hit_search_rs_explained_wo_stacktrace.json b/test_res/fixtures/one_hit_search_rs_explained_wo_stacktrace.json index efb70ce1..21d2c9a0 100644 --- a/test_res/fixtures/one_hit_search_rs_explained_wo_stacktrace.json +++ b/test_res/fixtures/one_hit_search_rs_explained_wo_stacktrace.json @@ -22,8 +22,8 @@ "issue_type": "AB001", "launch_name": "Launch 1", "log_level": 40000, - "original_message_lines": 28, - "original_message_words_number": 50, + "message_lines": 28, + "message_words_number": 50, "message": "java.lang.NullPointerException", "test_item": 1, "unique_id": "unique1", diff --git a/test_res/fixtures/one_hit_search_rs_merged.json b/test_res/fixtures/one_hit_search_rs_merged.json index d28a50ed..034a58ec 100644 --- a/test_res/fixtures/one_hit_search_rs_merged.json +++ b/test_res/fixtures/one_hit_search_rs_merged.json @@ -23,8 +23,8 @@ "issue_type": "AB001", "launch_name": "Launch 1", "log_level": 40000, - "original_message_lines": 1, - "original_message_words_number": 2, + "message_lines": 1, + "message_words_number": 2, "message": "", "test_item": 1, "start_time": "2020-01-15 10:57:43", diff --git a/test_res/fixtures/one_hit_search_rs_merged_wrong.json b/test_res/fixtures/one_hit_search_rs_merged_wrong.json index 3066c27d..4fc8351c 100644 --- a/test_res/fixtures/one_hit_search_rs_merged_wrong.json +++ b/test_res/fixtures/one_hit_search_rs_merged_wrong.json @@ -23,8 +23,8 @@ "issue_type": "AB001", "launch_name": "Launch 1", "log_level": 40000, - "original_message_lines": 1, - "original_message_words_number": 2, + "message_lines": 1, + "message_words_number": 2, "message": "", "test_item": 1, "start_time": "2020-01-15 10:57:43", diff --git a/test_res/fixtures/one_hit_search_rs_small_logs.json b/test_res/fixtures/one_hit_search_rs_small_logs.json index 92b54f97..e7807938 100644 --- a/test_res/fixtures/one_hit_search_rs_small_logs.json +++ b/test_res/fixtures/one_hit_search_rs_small_logs.json @@ -23,8 +23,8 @@ "issue_type": "AB001", "launch_name": "Launch 1", "log_level": 40000, - "original_message_lines": 1, - "original_message_words_number": 1, + "message_lines": 1, + "message_words_number": 1, "message": "message", "test_item": 1, "unique_id": "unique1", diff --git a/test_res/fixtures/search_logs_rq_first_group.json b/test_res/fixtures/search_logs_rq_first_group.json index bb56d322..ce94c09c 100644 --- a/test_res/fixtures/search_logs_rq_first_group.json +++ b/test_res/fixtures/search_logs_rq_first_group.json @@ -1 +1 @@ -{"query": {"function_score": {"boost_mode": "multiply", "functions": [{"exp": {"start_time": {"decay": 0.95, "offset": "1d", "origin": "2021-10-18 17:00:00", "scale": "7d"}}}, {"script_score": {"script": {"source": "0.2"}}}], "query": {"bool": {"filter": [{"range": {"log_level": {"gte": 40000}}}, {"exists": {"field": "issue_type"}}, {"term": {"is_merged": false}}, {"term": {"cluster_with_numbers": true}}], "must": [{"wildcard": {"cluster_message": "*"}}, {"more_like_this": {"boost": 1.0, "fields": ["whole_message"], "like": "error occurred error found error mined", "max_query_terms": 50, "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "5<95%"}}], "must_not": [{"term": {"test_item": {"boost": 1.0, "value": 2}}}], "should": [{"term": {"launch_id": 1}}, {"term": {"launch_name": "Launch name"}}]}}, "score_mode": "max"}}, "size": 10} \ No newline at end of file +{"_source": ["whole_message", "test_item", "is_merged", "detected_message", "launch_id", "cluster_id", "cluster_message", "potential_status_codes", "found_exceptions"], "size": 10, "query": {"function_score": {"boost_mode": "multiply", "functions": [{"exp": {"start_time": {"decay": 0.95, "offset": "1d", "origin": "2021-10-18 17:00:00", "scale": "7d"}}}, {"script_score": {"script": {"source": "0.2"}}}], "query": {"bool": {"filter": [{"range": {"log_level": {"gte": 40000}}}, {"exists": {"field": "issue_type"}}, {"term": {"is_merged": false}}, {"term": {"cluster_with_numbers": true}}], "must": [{"wildcard": {"cluster_message": "*"}}, {"more_like_this": {"boost": 1.0, "fields": ["whole_message"], "like": "error occurred error found error mined", "max_query_terms": 50, "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "5<95%"}}], "must_not": [{"term": {"test_item": {"boost": 1.0, "value": 2}}}], "should": [{"term": {"launch_id": 1}}, {"term": {"launch_name": "Launch name"}}]}}, "score_mode": "max"}}} \ No newline at end of file diff --git a/test_res/fixtures/search_logs_rq_first_group_2lines.json b/test_res/fixtures/search_logs_rq_first_group_2lines.json index 12852009..7886d369 100644 --- a/test_res/fixtures/search_logs_rq_first_group_2lines.json +++ b/test_res/fixtures/search_logs_rq_first_group_2lines.json @@ -1 +1 @@ -{"query": {"function_score": {"boost_mode": "multiply", "functions": [{"exp": {"start_time": {"decay": 0.95, "offset": "1d", "origin": "2021-10-18 17:00:00", "scale": "7d"}}}, {"script_score": {"script": {"source": "0.2"}}}], "query": {"bool": {"filter": [{"range": {"log_level": {"gte": 40000}}}, {"exists": {"field": "issue_type"}}, {"term": {"is_merged": false}}, {"term": {"cluster_with_numbers": true}}], "must": [{"wildcard": {"cluster_message": "*"}}, {"more_like_this": {"boost": 1.0, "fields": ["whole_message"], "like": "error occurred error found", "max_query_terms": 50, "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "5<95%"}}], "must_not": [{"term": {"test_item": {"boost": 1.0, "value": 2}}}], "should": [{"term": {"launch_id": 1}}, {"term": {"launch_name": "Launch name"}}]}}, "score_mode": "max"}}, "size": 10} \ No newline at end of file +{"_source": ["whole_message", "test_item", "is_merged", "detected_message", "launch_id", "cluster_id", "cluster_message", "potential_status_codes", "found_exceptions"], "size": 10, "query": {"function_score": {"boost_mode": "multiply", "functions": [{"exp": {"start_time": {"decay": 0.95, "offset": "1d", "origin": "2021-10-18 17:00:00", "scale": "7d"}}}, {"script_score": {"script": {"source": "0.2"}}}], "query": {"bool": {"filter": [{"range": {"log_level": {"gte": 40000}}}, {"exists": {"field": "issue_type"}}, {"term": {"is_merged": false}}, {"term": {"cluster_with_numbers": true}}], "must": [{"wildcard": {"cluster_message": "*"}}, {"more_like_this": {"boost": 1.0, "fields": ["whole_message"], "like": "error occurred error found", "max_query_terms": 50, "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "5<95%"}}], "must_not": [{"term": {"test_item": {"boost": 1.0, "value": 2}}}], "should": [{"term": {"launch_id": 1}}, {"term": {"launch_name": "Launch name"}}]}}, "score_mode": "max"}}} \ No newline at end of file diff --git a/test_res/fixtures/search_logs_rq_first_group_2lines_not_for_update.json b/test_res/fixtures/search_logs_rq_first_group_2lines_not_for_update.json index 367475e8..6e5c4f5c 100644 --- a/test_res/fixtures/search_logs_rq_first_group_2lines_not_for_update.json +++ b/test_res/fixtures/search_logs_rq_first_group_2lines_not_for_update.json @@ -1 +1 @@ -{"query": {"function_score": {"boost_mode": "multiply", "functions": [{"exp": {"start_time": {"decay": 0.95, "offset": "1d", "origin": "2021-10-18 17:00:00", "scale": "7d"}}}, {"script_score": {"script": {"source": "0.2"}}}], "query": {"bool": {"filter": [{"range": {"log_level": {"gte": 40000}}}, {"exists": {"field": "issue_type"}}, {"term": {"is_merged": false}}, {"term": {"cluster_with_numbers": true}}], "must": [{"wildcard": {"cluster_message": "*"}}, {"more_like_this": {"boost": 1.0, "fields": ["whole_message"], "like": "error occurred error found", "max_query_terms": 50, "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "5<95%"}}], "must_not": [{"term": {"test_item": {"boost": 1.0, "value": 2}}}, {"term": {"launch_id": 1}}], "should": [{"term": {"launch_name": "Launch name"}}]}}, "score_mode": "max"}}, "size": 10} \ No newline at end of file +{"_source": ["whole_message", "test_item", "is_merged", "detected_message", "launch_id", "cluster_id", "cluster_message", "potential_status_codes", "found_exceptions"], "size": 10, "query": {"function_score": {"boost_mode": "multiply", "functions": [{"exp": {"start_time": {"decay": 0.95, "offset": "1d", "origin": "2021-10-18 17:00:00", "scale": "7d"}}}, {"script_score": {"script": {"source": "0.2"}}}], "query": {"bool": {"filter": [{"range": {"log_level": {"gte": 40000}}}, {"exists": {"field": "issue_type"}}, {"term": {"is_merged": false}}, {"term": {"cluster_with_numbers": true}}], "must": [{"wildcard": {"cluster_message": "*"}}, {"more_like_this": {"boost": 1.0, "fields": ["whole_message"], "like": "error occurred error found", "max_query_terms": 50, "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "5<95%"}}], "must_not": [{"term": {"test_item": {"boost": 1.0, "value": 2}}}, {"term": {"launch_id": 1}}], "should": [{"term": {"launch_name": "Launch name"}}]}}, "score_mode": "max"}}} \ No newline at end of file diff --git a/test_res/fixtures/search_logs_rq_first_group_assertion_error.json b/test_res/fixtures/search_logs_rq_first_group_assertion_error.json index 82587e9c..d36ea70a 100644 --- a/test_res/fixtures/search_logs_rq_first_group_assertion_error.json +++ b/test_res/fixtures/search_logs_rq_first_group_assertion_error.json @@ -1 +1 @@ -{"query": {"function_score": {"boost_mode": "multiply", "functions": [{"exp": {"start_time": {"decay": 0.95, "offset": "1d", "origin": "2021-10-18 17:00:00", "scale": "7d"}}}, {"script_score": {"script": {"source": "0.2"}}}], "query": {"bool": {"filter": [{"range": {"log_level": {"gte": 40000}}}, {"exists": {"field": "issue_type"}}, {"term": {"is_merged": false}}, {"term": {"cluster_with_numbers": true}}], "must": [{"wildcard": {"cluster_message": "*"}}, {"more_like_this": {"boost": 1.0, "fields": ["whole_message"], "like": "assertionerror error occurred error found", "max_query_terms": 50, "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "5<95%"}}, {"more_like_this": {"boost": 1.0, "fields": ["found_exceptions"], "like": "AssertionError", "max_query_terms": 50, "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "1"}}], "must_not": [{"term": {"test_item": {"boost": 1.0, "value": 2}}}, {"term": {"launch_id": 1}}], "should": [{"term": {"launch_name": "Launch name"}}]}}, "score_mode": "max"}}, "size": 10} \ No newline at end of file +{"_source": ["whole_message", "test_item", "is_merged", "detected_message", "launch_id", "cluster_id", "cluster_message", "potential_status_codes", "found_exceptions"], "size": 10, "query": {"function_score": {"boost_mode": "multiply", "functions": [{"exp": {"start_time": {"decay": 0.95, "offset": "1d", "origin": "2021-10-18 17:00:00", "scale": "7d"}}}, {"script_score": {"script": {"source": "0.2"}}}], "query": {"bool": {"filter": [{"range": {"log_level": {"gte": 40000}}}, {"exists": {"field": "issue_type"}}, {"term": {"is_merged": false}}, {"term": {"cluster_with_numbers": true}}], "must": [{"wildcard": {"cluster_message": "*"}}, {"more_like_this": {"boost": 1.0, "fields": ["whole_message"], "like": "assertionerror error occurred error found", "max_query_terms": 50, "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "5<95%"}}, {"more_like_this": {"boost": 1.0, "fields": ["found_exceptions"], "like": "AssertionError", "max_query_terms": 50, "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "1"}}], "must_not": [{"term": {"test_item": {"boost": 1.0, "value": 2}}}, {"term": {"launch_id": 1}}], "should": [{"term": {"launch_name": "Launch name"}}]}}, "score_mode": "max"}}} \ No newline at end of file diff --git a/test_res/fixtures/search_logs_rq_first_group_assertion_error_status_code.json b/test_res/fixtures/search_logs_rq_first_group_assertion_error_status_code.json index fd722182..ede2334e 100644 --- a/test_res/fixtures/search_logs_rq_first_group_assertion_error_status_code.json +++ b/test_res/fixtures/search_logs_rq_first_group_assertion_error_status_code.json @@ -1 +1 @@ -{"query": {"function_score": {"boost_mode": "multiply", "functions": [{"exp": {"start_time": {"decay": 0.95, "offset": "1d", "origin": "2021-10-18 17:00:00", "scale": "7d"}}}, {"script_score": {"script": {"source": "0.2"}}}], "query": {"bool": {"filter": [{"range": {"log_level": {"gte": 40000}}}, {"exists": {"field": "issue_type"}}, {"term": {"is_merged": false}}, {"term": {"cluster_with_numbers": true}}], "must": [{"wildcard": {"cluster_message": "*"}}, {"more_like_this": {"boost": 1.0, "fields": ["whole_message"], "like": "assertionerror status code specialnumber error occurred error found", "max_query_terms": 50, "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "5<86%"}}, {"more_like_this": {"boost": 1.0, "fields": ["found_exceptions"], "like": "AssertionError", "max_query_terms": 50, "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "1"}}, {"more_like_this": {"boost": 1.0, "fields": ["potential_status_codes"], "like": "500", "max_query_terms": 50, "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "1"}}], "must_not": [{"term": {"test_item": {"boost": 1.0, "value": 5}}}, {"term": {"launch_id": 1}}], "should": [{"term": {"launch_name": "Launch name"}}]}}, "score_mode": "max"}}, "size": 10} \ No newline at end of file +{"_source": ["whole_message", "test_item", "is_merged", "detected_message", "launch_id", "cluster_id", "cluster_message", "potential_status_codes", "found_exceptions"], "size": 10, "query": {"function_score": {"boost_mode": "multiply", "functions": [{"exp": {"start_time": {"decay": 0.95, "offset": "1d", "origin": "2021-10-18 17:00:00", "scale": "7d"}}}, {"script_score": {"script": {"source": "0.2"}}}], "query": {"bool": {"filter": [{"range": {"log_level": {"gte": 40000}}}, {"exists": {"field": "issue_type"}}, {"term": {"is_merged": false}}, {"term": {"cluster_with_numbers": true}}], "must": [{"wildcard": {"cluster_message": "*"}}, {"more_like_this": {"boost": 1.0, "fields": ["whole_message"], "like": "assertionerror status code 500 error occurred error found", "max_query_terms": 50, "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "5<86%"}}, {"more_like_this": {"boost": 1.0, "fields": ["found_exceptions"], "like": "AssertionError", "max_query_terms": 50, "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "1"}}, {"more_like_this": {"boost": 1.0, "fields": ["potential_status_codes"], "like": "500", "max_query_terms": 50, "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "1"}}], "must_not": [{"term": {"test_item": {"boost": 1.0, "value": 5}}}, {"term": {"launch_id": 1}}], "should": [{"term": {"launch_name": "Launch name"}}]}}, "score_mode": "max"}}} \ No newline at end of file diff --git a/test_res/fixtures/search_logs_rq_first_group_no_such_element.json b/test_res/fixtures/search_logs_rq_first_group_no_such_element.json index 67c87e14..bd605f15 100644 --- a/test_res/fixtures/search_logs_rq_first_group_no_such_element.json +++ b/test_res/fixtures/search_logs_rq_first_group_no_such_element.json @@ -1 +1 @@ -{"query": {"function_score": {"boost_mode": "multiply", "functions": [{"exp": {"start_time": {"decay": 0.95, "offset": "1d", "origin": "2021-10-18 17:00:00", "scale": "7d"}}}, {"script_score": {"script": {"source": "0.2"}}}], "query": {"bool": {"filter": [{"range": {"log_level": {"gte": 40000}}}, {"exists": {"field": "issue_type"}}, {"term": {"is_merged": false}}, {"term": {"cluster_with_numbers": true}}], "must": [{"wildcard": {"cluster_message": "*"}}, {"more_like_this": {"boost": 1.0, "fields": ["whole_message"], "like": "nosuchelementexception error occurred error found", "max_query_terms": 50, "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "5<95%"}}, {"more_like_this": {"boost": 1.0, "fields": ["found_exceptions"], "like": "NoSuchElementException", "max_query_terms": 50, "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "1"}}], "must_not": [{"term": {"test_item": {"boost": 1.0, "value": 6}}}, {"term": {"launch_id": 1}}], "should": [{"term": {"launch_name": "Launch name"}}]}}, "score_mode": "max"}}, "size": 10} \ No newline at end of file +{"_source": ["whole_message", "test_item", "is_merged", "detected_message", "launch_id", "cluster_id", "cluster_message", "potential_status_codes", "found_exceptions"], "size": 10, "query": {"function_score": {"boost_mode": "multiply", "functions": [{"exp": {"start_time": {"decay": 0.95, "offset": "1d", "origin": "2021-10-18 17:00:00", "scale": "7d"}}}, {"script_score": {"script": {"source": "0.2"}}}], "query": {"bool": {"filter": [{"range": {"log_level": {"gte": 40000}}}, {"exists": {"field": "issue_type"}}, {"term": {"is_merged": false}}, {"term": {"cluster_with_numbers": true}}], "must": [{"wildcard": {"cluster_message": "*"}}, {"more_like_this": {"boost": 1.0, "fields": ["whole_message"], "like": "nosuchelementexception error occurred error found", "max_query_terms": 50, "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "5<95%"}}, {"more_like_this": {"boost": 1.0, "fields": ["found_exceptions"], "like": "NoSuchElementException", "max_query_terms": 50, "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "1"}}], "must_not": [{"term": {"test_item": {"boost": 1.0, "value": 6}}}, {"term": {"launch_id": 1}}], "should": [{"term": {"launch_name": "Launch name"}}]}}, "score_mode": "max"}}} \ No newline at end of file diff --git a/test_res/fixtures/search_logs_rq_first_group_no_such_element_all_log_lines.json b/test_res/fixtures/search_logs_rq_first_group_no_such_element_all_log_lines.json index 138731de..bd037575 100644 --- a/test_res/fixtures/search_logs_rq_first_group_no_such_element_all_log_lines.json +++ b/test_res/fixtures/search_logs_rq_first_group_no_such_element_all_log_lines.json @@ -1 +1 @@ -{"query": {"function_score": {"boost_mode": "multiply", "functions": [{"exp": {"start_time": {"decay": 0.95, "offset": "1d", "origin": "2021-10-18 17:00:00", "scale": "7d"}}}, {"script_score": {"script": {"source": "0.2"}}}], "query": {"bool": {"filter": [{"range": {"log_level": {"gte": 40000}}}, {"exists": {"field": "issue_type"}}, {"term": {"is_merged": false}}, {"term": {"cluster_with_numbers": true}}], "must": [{"wildcard": {"cluster_message": "*"}}, {"more_like_this": {"boost": 1.0, "fields": ["whole_message"], "like": "nosuchelementexception error occurred error found assert query", "max_query_terms": 50, "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "5<83%"}}, {"more_like_this": {"boost": 1.0, "fields": ["found_exceptions"], "like": "NoSuchElementException", "max_query_terms": 50, "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "1"}}], "must_not": [{"term": {"test_item": {"boost": 1.0, "value": 6}}}, {"term": {"launch_id": 1}}], "should": [{"term": {"launch_name": "Launch name"}}]}}, "score_mode": "max"}}, "size": 10} \ No newline at end of file +{"_source": ["whole_message", "test_item", "is_merged", "detected_message", "launch_id", "cluster_id", "cluster_message", "potential_status_codes", "found_exceptions"], "size": 10, "query": {"function_score": {"boost_mode": "multiply", "functions": [{"exp": {"start_time": {"decay": 0.95, "offset": "1d", "origin": "2021-10-18 17:00:00", "scale": "7d"}}}, {"script_score": {"script": {"source": "0.2"}}}], "query": {"bool": {"filter": [{"range": {"log_level": {"gte": 40000}}}, {"exists": {"field": "issue_type"}}, {"term": {"is_merged": false}}, {"term": {"cluster_with_numbers": true}}], "must": [{"wildcard": {"cluster_message": "*"}}, {"more_like_this": {"boost": 1.0, "fields": ["whole_message"], "like": "nosuchelementexception error occurred error found assert query", "max_query_terms": 50, "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "5<83%"}}, {"more_like_this": {"boost": 1.0, "fields": ["found_exceptions"], "like": "NoSuchElementException", "max_query_terms": 50, "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "1"}}], "must_not": [{"term": {"test_item": {"boost": 1.0, "value": 6}}}, {"term": {"launch_id": 1}}], "should": [{"term": {"launch_name": "Launch name"}}]}}, "score_mode": "max"}}} \ No newline at end of file diff --git a/test_res/fixtures/search_logs_rq_first_group_not_for_update.json b/test_res/fixtures/search_logs_rq_first_group_not_for_update.json index 39c9ff5d..bf3e8e8a 100644 --- a/test_res/fixtures/search_logs_rq_first_group_not_for_update.json +++ b/test_res/fixtures/search_logs_rq_first_group_not_for_update.json @@ -1 +1 @@ -{"query": {"function_score": {"boost_mode": "multiply", "functions": [{"exp": {"start_time": {"decay": 0.95, "offset": "1d", "origin": "2021-10-18 17:00:00", "scale": "7d"}}}, {"script_score": {"script": {"source": "0.2"}}}], "query": {"bool": {"filter": [{"range": {"log_level": {"gte": 40000}}}, {"exists": {"field": "issue_type"}}, {"term": {"is_merged": false}}, {"term": {"cluster_with_numbers": true}}], "must": [{"wildcard": {"cluster_message": "*"}}, {"more_like_this": {"boost": 1.0, "fields": ["whole_message"], "like": "error occurred error found error mined", "max_query_terms": 50, "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "5<95%"}}], "must_not": [{"term": {"test_item": {"boost": 1.0, "value": 2}}}, {"term": {"launch_id": 1}}], "should": [{"term": {"launch_name": "Launch name"}}]}}, "score_mode": "max"}}, "size": 10} \ No newline at end of file +{"_source": ["whole_message", "test_item", "is_merged", "detected_message", "launch_id", "cluster_id", "cluster_message", "potential_status_codes", "found_exceptions"], "size": 10, "query": {"function_score": {"boost_mode": "multiply", "functions": [{"exp": {"start_time": {"decay": 0.95, "offset": "1d", "origin": "2021-10-18 17:00:00", "scale": "7d"}}}, {"script_score": {"script": {"source": "0.2"}}}], "query": {"bool": {"filter": [{"range": {"log_level": {"gte": 40000}}}, {"exists": {"field": "issue_type"}}, {"term": {"is_merged": false}}, {"term": {"cluster_with_numbers": true}}], "must": [{"wildcard": {"cluster_message": "*"}}, {"more_like_this": {"boost": 1.0, "fields": ["whole_message"], "like": "error occurred error found error mined", "max_query_terms": 50, "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "5<95%"}}], "must_not": [{"term": {"test_item": {"boost": 1.0, "value": 2}}}, {"term": {"launch_id": 1}}], "should": [{"term": {"launch_name": "Launch name"}}]}}, "score_mode": "max"}}} \ No newline at end of file diff --git a/test_res/fixtures/search_logs_rq_first_group_small_logs.json b/test_res/fixtures/search_logs_rq_first_group_small_logs.json index 2e7f79c5..27b87ab4 100644 --- a/test_res/fixtures/search_logs_rq_first_group_small_logs.json +++ b/test_res/fixtures/search_logs_rq_first_group_small_logs.json @@ -1 +1 @@ -{"query": {"function_score": {"boost_mode": "multiply", "functions": [{"exp": {"start_time": {"decay": 0.95, "offset": "1d", "origin": "2021-10-18 17:00:00", "scale": "7d"}}}, {"script_score": {"script": {"source": "0.2"}}}], "query": {"bool": {"filter": [{"range": {"log_level": {"gte": 40000}}}, {"exists": {"field": "issue_type"}}, {"term": {"is_merged": true}}, {"term": {"cluster_with_numbers": true}}], "must": [{"wildcard": {"cluster_message": "*"}}, {"more_like_this": {"boost": 1.0, "fields": ["whole_message"], "like": "error occurred twice assertionerror error occurred error found", "max_query_terms": 50, "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "5<80%"}}, {"more_like_this": {"boost": 1.0, "fields": ["found_exceptions"], "like": "assertionerror", "max_query_terms": 50, "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "1"}}], "must_not": [{"term": {"test_item": {"boost": 1.0, "value": 2}}}, {"term": {"launch_id": 1}}], "should": [{"term": {"launch_name": "Launch name"}}]}}, "score_mode": "max"}}, "size": 10} \ No newline at end of file +{"_source": ["whole_message", "test_item", "is_merged", "detected_message", "launch_id", "cluster_id", "cluster_message", "potential_status_codes", "found_exceptions"], "size": 10, "query": {"function_score": {"boost_mode": "multiply", "functions": [{"exp": {"start_time": {"decay": 0.95, "offset": "1d", "origin": "2021-10-18 17:00:00", "scale": "7d"}}}, {"script_score": {"script": {"source": "0.2"}}}], "query": {"bool": {"filter": [{"range": {"log_level": {"gte": 40000}}}, {"exists": {"field": "issue_type"}}, {"term": {"is_merged": true}}, {"term": {"cluster_with_numbers": true}}], "must": [{"wildcard": {"cluster_message": "*"}}, {"more_like_this": {"boost": 1.0, "fields": ["whole_message"], "like": "error occurred twice assertionerror error occurred error found", "max_query_terms": 50, "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "5<80%"}}, {"more_like_this": {"boost": 1.0, "fields": ["found_exceptions"], "like": "assertionerror", "max_query_terms": 50, "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "1"}}], "must_not": [{"term": {"test_item": {"boost": 1.0, "value": 2}}}, {"term": {"launch_id": 1}}], "should": [{"term": {"launch_name": "Launch name"}}]}}, "score_mode": "max"}}} \ No newline at end of file diff --git a/test_res/fixtures/search_logs_rq_second_group.json b/test_res/fixtures/search_logs_rq_second_group.json index 1cbb4206..fddf4d8d 100644 --- a/test_res/fixtures/search_logs_rq_second_group.json +++ b/test_res/fixtures/search_logs_rq_second_group.json @@ -1 +1 @@ -{"query": {"function_score": {"boost_mode": "multiply", "functions": [{"exp": {"start_time": {"decay": 0.95, "offset": "1d", "origin": "2021-10-18 17:00:00", "scale": "7d"}}}, {"script_score": {"script": {"source": "0.2"}}}], "query": {"bool": {"filter": [{"range": {"log_level": {"gte": 40000}}}, {"exists": {"field": "issue_type"}}, {"term": {"is_merged": false}}, {"term": {"cluster_with_numbers": true}}], "must": [{"wildcard": {"cluster_message": "*"}}, {"more_like_this": {"boost": 1.0, "fields": ["whole_message"], "like": "error occurred error found assert query", "max_query_terms": 50, "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "5<80%"}}], "must_not": [{"term": {"test_item": {"boost": 1.0, "value": 6}}}], "should": [{"term": {"launch_id": 1}}, {"term": {"launch_name": "Launch name"}}]}}, "score_mode": "max"}}, "size": 10} \ No newline at end of file +{"_source": ["whole_message", "test_item", "is_merged", "detected_message", "launch_id", "cluster_id", "cluster_message", "potential_status_codes", "found_exceptions"], "size": 10, "query": {"function_score": {"boost_mode": "multiply", "functions": [{"exp": {"start_time": {"decay": 0.95, "offset": "1d", "origin": "2021-10-18 17:00:00", "scale": "7d"}}}, {"script_score": {"script": {"source": "0.2"}}}], "query": {"bool": {"filter": [{"range": {"log_level": {"gte": 40000}}}, {"exists": {"field": "issue_type"}}, {"term": {"is_merged": false}}, {"term": {"cluster_with_numbers": true}}], "must": [{"wildcard": {"cluster_message": "*"}}, {"more_like_this": {"boost": 1.0, "fields": ["whole_message"], "like": "error occurred error found assert query", "max_query_terms": 50, "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "5<80%"}}], "must_not": [{"term": {"test_item": {"boost": 1.0, "value": 6}}}], "should": [{"term": {"launch_id": 1}}, {"term": {"launch_name": "Launch name"}}]}}, "score_mode": "max"}}} \ No newline at end of file diff --git a/test_res/fixtures/search_logs_rq_second_group_not_for_update.json b/test_res/fixtures/search_logs_rq_second_group_not_for_update.json index e0abe548..c08895d3 100644 --- a/test_res/fixtures/search_logs_rq_second_group_not_for_update.json +++ b/test_res/fixtures/search_logs_rq_second_group_not_for_update.json @@ -1 +1 @@ -{"query": {"function_score": {"boost_mode": "multiply", "functions": [{"exp": {"start_time": {"decay": 0.95, "offset": "1d", "origin": "2021-10-18 17:00:00", "scale": "7d"}}}, {"script_score": {"script": {"source": "0.2"}}}], "query": {"bool": {"filter": [{"range": {"log_level": {"gte": 40000}}}, {"exists": {"field": "issue_type"}}, {"term": {"is_merged": false}}, {"term": {"cluster_with_numbers": true}}], "must": [{"wildcard": {"cluster_message": "*"}}, {"more_like_this": {"boost": 1.0, "fields": ["whole_message"], "like": "error occurred error found assert query", "max_query_terms": 50, "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "5<80%"}}], "must_not": [{"term": {"test_item": {"boost": 1.0, "value": 6}}}, {"term": {"launch_id": 1}}], "should": [{"term": {"launch_name": "Launch name"}}]}}, "score_mode": "max"}}, "size": 10} \ No newline at end of file +{"_source": ["whole_message", "test_item", "is_merged", "detected_message", "launch_id", "cluster_id", "cluster_message", "potential_status_codes", "found_exceptions"], "size": 10, "query": {"function_score": {"boost_mode": "multiply", "functions": [{"exp": {"start_time": {"decay": 0.95, "offset": "1d", "origin": "2021-10-18 17:00:00", "scale": "7d"}}}, {"script_score": {"script": {"source": "0.2"}}}], "query": {"bool": {"filter": [{"range": {"log_level": {"gte": 40000}}}, {"exists": {"field": "issue_type"}}, {"term": {"is_merged": false}}, {"term": {"cluster_with_numbers": true}}], "must": [{"wildcard": {"cluster_message": "*"}}, {"more_like_this": {"boost": 1.0, "fields": ["whole_message"], "like": "error occurred error found assert query", "max_query_terms": 50, "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "5<80%"}}], "must_not": [{"term": {"test_item": {"boost": 1.0, "value": 6}}}, {"term": {"launch_id": 1}}], "should": [{"term": {"launch_name": "Launch name"}}]}}, "score_mode": "max"}}} \ No newline at end of file diff --git a/test_res/fixtures/search_logs_rq_second_group_small_logs.json b/test_res/fixtures/search_logs_rq_second_group_small_logs.json index cdec6135..8685804b 100644 --- a/test_res/fixtures/search_logs_rq_second_group_small_logs.json +++ b/test_res/fixtures/search_logs_rq_second_group_small_logs.json @@ -1 +1 @@ -{"size": 10, "query": {"function_score": {"query": {"bool": {"filter": [{"range": {"log_level": {"gte": 40000}}}, {"exists": {"field": "issue_type"}}, {"term": {"is_merged": true}}, {"term": {"cluster_with_numbers": true}}], "must_not": [{"term": {"test_item": {"value": 5, "boost": 1.0}}}, {"term": {"launch_id": 1}}], "should": [{"term": {"launch_name": "Launch name"}}], "must": [{"wildcard": {"cluster_message": "*"}}, {"more_like_this": {"fields": ["whole_message"], "like": "assertionerror status code specialnumber error occurred", "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "5<83%", "max_query_terms": 50, "boost": 1.0}}, {"more_like_this": {"fields": ["found_exceptions"], "like": "assertionerror", "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "1", "max_query_terms": 50, "boost": 1.0}}, {"more_like_this": {"fields": ["potential_status_codes"], "like": "500", "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "1", "max_query_terms": 50, "boost": 1.0}}]}}, "functions": [{"exp": {"start_time": {"origin": "2021-10-18 17:00:00", "scale": "7d", "offset": "1d", "decay": 0.95}}}, {"script_score": {"script": {"source": "0.2"}}}], "score_mode": "max", "boost_mode": "multiply"}}} \ No newline at end of file +{"_source": ["whole_message", "test_item", "is_merged", "detected_message", "launch_id", "cluster_id", "cluster_message", "potential_status_codes", "found_exceptions"], "size": 10, "query": {"function_score": {"query": {"bool": {"filter": [{"range": {"log_level": {"gte": 40000}}}, {"exists": {"field": "issue_type"}}, {"term": {"is_merged": true}}, {"term": {"cluster_with_numbers": true}}], "must_not": [{"term": {"test_item": {"value": 5, "boost": 1.0}}}, {"term": {"launch_id": 1}}], "should": [{"term": {"launch_name": "Launch name"}}], "must": [{"wildcard": {"cluster_message": "*"}}, {"more_like_this": {"fields": ["whole_message"], "like": "assertionerror status code 500 error occurred", "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "5<83%", "max_query_terms": 50, "boost": 1.0}}, {"more_like_this": {"fields": ["found_exceptions"], "like": "assertionerror", "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "1", "max_query_terms": 50, "boost": 1.0}}, {"more_like_this": {"fields": ["potential_status_codes"], "like": "500", "min_doc_freq": 1, "min_term_freq": 1, "minimum_should_match": "1", "max_query_terms": 50, "boost": 1.0}}]}}, "functions": [{"exp": {"start_time": {"origin": "2021-10-18 17:00:00", "scale": "7d", "offset": "1d", "decay": 0.95}}}, {"script_score": {"script": {"source": "0.2"}}}], "score_mode": "max", "boost_mode": "multiply"}}} \ No newline at end of file diff --git a/test_res/fixtures/three_hits_search_rs.json b/test_res/fixtures/three_hits_search_rs.json index 77387ff2..30a62b28 100644 --- a/test_res/fixtures/three_hits_search_rs.json +++ b/test_res/fixtures/three_hits_search_rs.json @@ -23,8 +23,8 @@ "issue_type": "AB001", "launch_name": "Launch 1", "log_level": 40000, - "original_message_lines": 1, - "original_message_words_number": 2, + "message_lines": 1, + "message_words_number": 2, "message": "Message SPECIALNUMBER http : localhost/admin \n java.lang.NoClassDefFoundError\r", "test_item": 1, "start_time": "2020-01-15 10:57:43", @@ -60,8 +60,8 @@ "issue_type": "PB001", "launch_name": "Launch 1", "log_level": 40000, - "original_message_lines": 1, - "original_message_words_number": 2, + "message_lines": 1, + "message_words_number": 2, "message": "Message SPECIALNUMBER http : localhost/admin \n java.lang.NoClassDefFoundError\r", "test_item": 2, "start_time": "2020-01-15 10:57:43", @@ -97,8 +97,8 @@ "issue_type": "PB001", "launch_name": "Launch 1", "log_level": 40000, - "original_message_lines": 1, - "original_message_words_number": 2, + "message_lines": 1, + "message_words_number": 2, "message": "Message SPECIALNUMBER http : localhost/admin \n java.lang.NoClassDefFoundError\r", "test_item": 3, "start_time": "2020-01-15 10:57:43", diff --git a/test_res/fixtures/three_hits_search_rs_explained.json b/test_res/fixtures/three_hits_search_rs_explained.json index ea18c728..921e7025 100644 --- a/test_res/fixtures/three_hits_search_rs_explained.json +++ b/test_res/fixtures/three_hits_search_rs_explained.json @@ -22,8 +22,8 @@ "issue_type": "PB001", "launch_name": "Launch 1", "log_level": 40000, - "original_message_lines": 28, - "original_message_words_number": 50, + "message_lines": 28, + "message_words_number": 50, "test_item_name": "first test", "message": "java.lang.NullPointerException\n\tat tv.horizon.web.ta.ui.elements.assets.Promotion.setFeed(Promotion.java:)\n\tat tv.horizon.web.ta.ui.elements.assets.Promotion.(Promotion.java:)\n\tat tv.horizon.web.ta.services.navigation.AssetsManager.checkTitleCardForPromotions(AssetsManager.java:)\n\tat tv.horizon.web.tests.checklist.AspotAndPromotionChecklist.checkPromotionFeeds(AspotAndPromotionChecklist.java:)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:)\n\tat java.lang.reflect.Method.invoke(Method.java:)\n\tat org.testng.internal.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:)\n\tat org.testng.internal.MethodInvocationHelper$.runTestMethod(MethodInvocationHelper.java:)\n\tat org.springframework.test.context.testng.AbstractTestNGSpringContextTests.run(AbstractTestNGSpringContextTests.java:)\n\tat org.testng.internal.MethodInvocationHelper.invokeHookable(MethodInvocationHelper.java:)\n\tat org.testng.internal.Invoker.invokeMethod(Invoker.java:)\n\tat org.testng.internal.Invoker.invokeTestMethod(Invoker.java:)\n\tat org.testng.internal.Invoker.invokeTestMethods(Invoker.java:)\n\tat org.testng.internal.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:)\n\tat org.testng.internal.TestMethodWorker.run(TestMethodWorker.java:)\n\tat org.testng.TestRunner.privateRun(TestRunner.java:)\n\tat org.testng.TestRunner.run(TestRunner.java:)\n\tat org.testng.SuiteRunner.runTest(SuiteRunner.java:)\n\tat org.testng.SuiteRunner.access$(SuiteRunner.java:)\n\tat org.testng.SuiteRunner$SuiteWorker.run(SuiteRunner.java:)\n\tat org.testng.internal.thread.ThreadUtil$.call(ThreadUtil.java:)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:)\n\tat java.lang.Thread.run(Thread.java:)\n", "test_item": 2, @@ -45,8 +45,8 @@ "issue_type": "AB001", "launch_name": "Launch 1", "log_level": 40000, - "original_message_lines": 29, - "original_message_words_number": 51, + "message_lines": 29, + "message_words_number": 51, "message": "java.lang.NullPointerException\n\tat tv.horizon.web.ta.ui.elements.assets.Promotion.setFeed(Promotion.java:)\n\tat tv.horizon.web.ta.ui.elements.assets.Promotion.(Promotion.java:)\n\tat tv.horizon.web.ta.ui.elements.aspot.Aspot.isMostWatchItemShouldBeDisplayed(Aspot.java:)\n\tat tv.horizon.web.tests.checklist.AspotAndPromotionChecklist.checkAspot(AspotAndPromotionChecklist.java:)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:)\n\tat java.lang.reflect.Method.invoke(Method.java:)\n\tat org.testng.internal.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:)\n\tat org.testng.internal.MethodInvocationHelper$.runTestMethod(MethodInvocationHelper.java:)\n\tat org.springframework.test.context.testng.AbstractTestNGSpringContextTests.run(AbstractTestNGSpringContextTests.java:)\n\tat org.testng.internal.MethodInvocationHelper.invokeHookable(MethodInvocationHelper.java:)\n\tat org.testng.internal.Invoker.invokeMethod(Invoker.java:)\n\tat org.testng.internal.Invoker.invokeTestMethod(Invoker.java:)\n\tat org.testng.internal.Invoker.invokeTestMethods(Invoker.java:)\n\tat org.testng.internal.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:)\n\tat org.testng.internal.TestMethodWorker.run(TestMethodWorker.java:)\n\tat org.testng.TestRunner.privateRun(TestRunner.java:)\n\tat org.testng.TestRunner.run(TestRunner.java:)\n\tat org.testng.SuiteRunner.runTest(SuiteRunner.java:)\n\tat org.testng.SuiteRunner.access$(SuiteRunner.java:)\n\tat org.testng.SuiteRunner$SuiteWorker.run(SuiteRunner.java:)\n\tat org.testng.internal.thread.ThreadUtil$.call(ThreadUtil.java:)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:)\n\tat java.lang.Thread.run(Thread.java:)\n", "test_item": 1, "test_item_name": "first test 2", @@ -68,8 +68,8 @@ "issue_type": "PB001", "launch_name": "Launch 1", "log_level": 40000, - "original_message_lines": 29, - "original_message_words_number": 51, + "message_lines": 29, + "message_words_number": 51, "message": "java.lang.NullPointerException status\n\tat tv.horizon.web.ta.ui.elements.assets.Promotion.setFeed(Promotion.java:)\n\tat tv.horizon.web.ta.ui.elements.assets.Promotion.(Promotion.java:)\n\tat tv.horizon.web.ta.ui.elements.aspot.Aspot.isMostWatchItemShouldBeDisplayed(Aspot.java:)\n\tat tv.horizon.web.tests.checklist.AspotAndPromotionChecklist.checkAspot(AspotAndPromotionChecklist.java:)\n\tat tv.horizon.web.tests.checklist.AspotAndPromotionChecklist.calculate(AspotAndPromotionChecklist.java:)\n\tat tv.horizon.web.tests.checklist.AspotAndPromotionChecklist.evaluate(AspotAndPromotionChecklist.java:)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:)\n\tat java.lang.reflect.Method.invoke(Method.java:)\n\tat org.testng.internal.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:)\n\tat org.testng.internal.MethodInvocationHelper$.runTestMethod(MethodInvocationHelper.java:)\n\tat org.springframework.test.context.testng.AbstractTestNGSpringContextTests.run(AbstractTestNGSpringContextTests.java:)\n\tat org.testng.internal.MethodInvocationHelper.invokeHookable(MethodInvocationHelper.java:)\n\tat org.testng.internal.Invoker.invokeMethod(Invoker.java:)\n\tat org.testng.internal.Invoker.invokeTestMethod(Invoker.java:)\n\tat org.testng.internal.Invoker.invokeTestMethods(Invoker.java:)\n\tat org.testng.internal.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:)\n\tat org.testng.internal.TestMethodWorker.run(TestMethodWorker.java:)\n\tat org.testng.TestRunner.privateRun(TestRunner.java:)\n\tat org.testng.TestRunner.run(TestRunner.java:)\n\tat org.testng.SuiteRunner.runTest(SuiteRunner.java:)\n\tat org.testng.SuiteRunner.access$(SuiteRunner.java:)\n\tat org.testng.SuiteRunner$SuiteWorker.run(SuiteRunner.java:)\n\tat org.testng.internal.thread.ThreadUtil$.call(ThreadUtil.java:)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:)\n\tat java.lang.Thread.run(Thread.java:)\n", "test_item": 3, "test_item_name": "first test new", diff --git a/test_res/fixtures/three_hits_search_rs_for_cluster_suggestions.json b/test_res/fixtures/three_hits_search_rs_for_cluster_suggestions.json index 9ac0e765..ee3e4a90 100644 --- a/test_res/fixtures/three_hits_search_rs_for_cluster_suggestions.json +++ b/test_res/fixtures/three_hits_search_rs_for_cluster_suggestions.json @@ -22,8 +22,8 @@ "issue_type": "PB001", "launch_name": "Launch 1", "log_level": 40000, - "original_message_lines": 28, - "original_message_words_number": 50, + "message_lines": 28, + "message_words_number": 50, "test_item_name": "first test", "message": "java.lang.NullPointerException\n\tat tv.horizon.web.ta.ui.elements.assets.Promotion.setFeed(Promotion.java:)\n\tat tv.horizon.web.ta.ui.elements.assets.Promotion.(Promotion.java:)\n\tat tv.horizon.web.ta.services.navigation.AssetsManager.checkTitleCardForPromotions(AssetsManager.java:)\n\tat tv.horizon.web.tests.checklist.AspotAndPromotionChecklist.checkPromotionFeeds(AspotAndPromotionChecklist.java:)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:)\n\tat java.lang.reflect.Method.invoke(Method.java:)\n\tat org.testng.internal.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:)\n\tat org.testng.internal.MethodInvocationHelper$.runTestMethod(MethodInvocationHelper.java:)\n\tat org.springframework.test.context.testng.AbstractTestNGSpringContextTests.run(AbstractTestNGSpringContextTests.java:)\n\tat org.testng.internal.MethodInvocationHelper.invokeHookable(MethodInvocationHelper.java:)\n\tat org.testng.internal.Invoker.invokeMethod(Invoker.java:)\n\tat org.testng.internal.Invoker.invokeTestMethod(Invoker.java:)\n\tat org.testng.internal.Invoker.invokeTestMethods(Invoker.java:)\n\tat org.testng.internal.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:)\n\tat org.testng.internal.TestMethodWorker.run(TestMethodWorker.java:)\n\tat org.testng.TestRunner.privateRun(TestRunner.java:)\n\tat org.testng.TestRunner.run(TestRunner.java:)\n\tat org.testng.SuiteRunner.runTest(SuiteRunner.java:)\n\tat org.testng.SuiteRunner.access$(SuiteRunner.java:)\n\tat org.testng.SuiteRunner$SuiteWorker.run(SuiteRunner.java:)\n\tat org.testng.internal.thread.ThreadUtil$.call(ThreadUtil.java:)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:)\n\tat java.lang.Thread.run(Thread.java:)\n", "test_item": 2, @@ -45,8 +45,8 @@ "issue_type": "AB001", "launch_name": "Launch 1", "log_level": 40000, - "original_message_lines": 29, - "original_message_words_number": 51, + "message_lines": 29, + "message_words_number": 51, "message": "java.lang.NullPointerException\n\tat tv.horizon.web.ta.ui.elements.assets.Promotion.setFeed(Promotion.java:)\n\tat tv.horizon.web.ta.ui.elements.assets.Promotion.(Promotion.java:)\n\tat tv.horizon.web.ta.ui.elements.aspot.Aspot.isMostWatchItemShouldBeDisplayed(Aspot.java:)\n\tat tv.horizon.web.tests.checklist.AspotAndPromotionChecklist.checkAspot(AspotAndPromotionChecklist.java:)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:)\n\tat java.lang.reflect.Method.invoke(Method.java:)\n\tat org.testng.internal.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:)\n\tat org.testng.internal.MethodInvocationHelper$.runTestMethod(MethodInvocationHelper.java:)\n\tat org.springframework.test.context.testng.AbstractTestNGSpringContextTests.run(AbstractTestNGSpringContextTests.java:)\n\tat org.testng.internal.MethodInvocationHelper.invokeHookable(MethodInvocationHelper.java:)\n\tat org.testng.internal.Invoker.invokeMethod(Invoker.java:)\n\tat org.testng.internal.Invoker.invokeTestMethod(Invoker.java:)\n\tat org.testng.internal.Invoker.invokeTestMethods(Invoker.java:)\n\tat org.testng.internal.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:)\n\tat org.testng.internal.TestMethodWorker.run(TestMethodWorker.java:)\n\tat org.testng.TestRunner.privateRun(TestRunner.java:)\n\tat org.testng.TestRunner.run(TestRunner.java:)\n\tat org.testng.SuiteRunner.runTest(SuiteRunner.java:)\n\tat org.testng.SuiteRunner.access$(SuiteRunner.java:)\n\tat org.testng.SuiteRunner$SuiteWorker.run(SuiteRunner.java:)\n\tat org.testng.internal.thread.ThreadUtil$.call(ThreadUtil.java:)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:)\n\tat java.lang.Thread.run(Thread.java:)\n", "test_item": 1, "test_item_name": "first test 2", @@ -68,8 +68,8 @@ "issue_type": "PB001", "launch_name": "Launch 1", "log_level": 40000, - "original_message_lines": 29, - "original_message_words_number": 51, + "message_lines": 29, + "message_words_number": 51, "message": "java.lang.NullPointerException\n\tat tv.horizon.web.ta.ui.elements.assets.Promotion.setFeed(Promotion.java:)\n\tat tv.horizon.web.ta.ui.elements.assets.Promotion.(Promotion.java:)\n\tat tv.horizon.web.ta.ui.elements.aspot.Aspot.isMostWatchItemShouldBeDisplayed(Aspot.java:)\n\tat tv.horizon.web.tests.checklist.AspotAndPromotionChecklist.checkAspot(AspotAndPromotionChecklist.java:)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:)\n\tat java.lang.reflect.Method.invoke(Method.java:)\n\tat org.testng.internal.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:)\n\tat org.testng.internal.MethodInvocationHelper$.runTestMethod(MethodInvocationHelper.java:)\n\tat org.springframework.test.context.testng.AbstractTestNGSpringContextTests.run(AbstractTestNGSpringContextTests.java:)\n\tat org.testng.internal.MethodInvocationHelper.invokeHookable(MethodInvocationHelper.java:)\n\tat org.testng.internal.Invoker.invokeMethod(Invoker.java:)\n\tat org.testng.internal.Invoker.invokeTestMethod(Invoker.java:)\n\tat org.testng.internal.Invoker.invokeTestMethods(Invoker.java:)\n\tat org.testng.internal.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:)\n\tat org.testng.internal.TestMethodWorker.run(TestMethodWorker.java:)\n\tat org.testng.TestRunner.privateRun(TestRunner.java:)\n\tat org.testng.TestRunner.run(TestRunner.java:)\n\tat org.testng.SuiteRunner.runTest(SuiteRunner.java:)\n\tat org.testng.SuiteRunner.access$(SuiteRunner.java:)\n\tat org.testng.SuiteRunner$SuiteWorker.run(SuiteRunner.java:)\n\tat org.testng.internal.thread.ThreadUtil$.call(ThreadUtil.java:)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:)\n\tat java.lang.Thread.run(Thread.java:)\n", "test_item": 3, "test_item_name": "first test new", diff --git a/test_res/fixtures/three_hits_search_rs_with_duplicate.json b/test_res/fixtures/three_hits_search_rs_with_duplicate.json index 2f0fdd5f..1043fbad 100644 --- a/test_res/fixtures/three_hits_search_rs_with_duplicate.json +++ b/test_res/fixtures/three_hits_search_rs_with_duplicate.json @@ -22,8 +22,8 @@ "issue_type": "AB001", "launch_name": "Launch 1", "log_level": 40000, - "original_message_lines": 1, - "original_message_words_number": 2, + "message_lines": 1, + "message_words_number": 2, "message": "Message SPECIALNUMBER http : localhost/admin \n java.lang.NoClassDefFoundError\r", "test_item": 3, "start_time": "2020-01-17 10:57:43", @@ -57,8 +57,8 @@ "issue_type": "PB001", "launch_name": "Launch 1", "log_level": 40000, - "original_message_lines": 1, - "original_message_words_number": 2, + "message_lines": 1, + "message_words_number": 2, "message": "Message SPECIALNUMBER http : localhost/admin \n java.lang.NoClassDefFoundError\r", "test_item": 2, "start_time": "2020-01-15 10:57:43", @@ -92,8 +92,8 @@ "issue_type": "PB001", "launch_name": "Launch 1", "log_level": 40000, - "original_message_lines": 1, - "original_message_words_number": 2, + "message_lines": 1, + "message_words_number": 2, "message": "Message SPECIALNUMBER http : localhost/admin \n java.lang.NoClassDefFoundError\r", "test_item": 1, "start_time": "2020-01-15 10:57:43", diff --git a/test_res/fixtures/three_hits_search_rs_with_one_unique_id.json b/test_res/fixtures/three_hits_search_rs_with_one_unique_id.json index 3bcb96ef..d0419130 100644 --- a/test_res/fixtures/three_hits_search_rs_with_one_unique_id.json +++ b/test_res/fixtures/three_hits_search_rs_with_one_unique_id.json @@ -22,8 +22,8 @@ "issue_type": "AB001", "launch_name": "Launch 1", "log_level": 40000, - "original_message_lines": 1, - "original_message_words_number": 2, + "message_lines": 1, + "message_words_number": 2, "message": "Message http : localhost/admin \n java.lang.NoClassDefFoundError\r", "test_item": 1, "start_time": "2020-01-15 10:57:43", @@ -56,8 +56,8 @@ "issue_type": "PB001", "launch_name": "Launch 1", "log_level": 40000, - "original_message_lines": 1, - "original_message_words_number": 2, + "message_lines": 1, + "message_words_number": 2, "message": "Message http : localhost/admin \n java.lang.NoClassDefFoundError\r", "test_item": 2, "start_time": "2020-01-14 10:57:43", @@ -90,8 +90,8 @@ "issue_type": "PB001", "launch_name": "Launch 1", "log_level": 40000, - "original_message_lines": 1, - "original_message_words_number": 2, + "message_lines": 1, + "message_words_number": 2, "message": "Message http : localhost/admin \n java.lang.NoClassDefFoundError\r", "test_item": 3, "start_time": "2020-01-13 10:57:43", diff --git a/test_res/fixtures/three_hits_with_no_defect.json b/test_res/fixtures/three_hits_with_no_defect.json index 2cf001c3..3f424c31 100644 --- a/test_res/fixtures/three_hits_with_no_defect.json +++ b/test_res/fixtures/three_hits_with_no_defect.json @@ -22,8 +22,8 @@ "issue_type": "AB001", "launch_name": "Launch 1", "log_level": 40000, - "original_message_lines": 1, - "original_message_words_number": 2, + "message_lines": 1, + "message_words_number": 2, "message": "Message http : localhost/admin \n java.lang.NoClassDefFoundError\r", "test_item": 1, "start_time": "2020-01-15 10:57:43", @@ -58,8 +58,8 @@ "issue_type": "ND001", "launch_name": "Launch 1", "log_level": 40000, - "original_message_lines": 1, - "original_message_words_number": 2, + "message_lines": 1, + "message_words_number": 2, "message": "Message http : localhost/admin \n java.lang.NoClassDefFoundError\r", "test_item": 2, "start_time": "2020-01-15 10:57:43", @@ -94,8 +94,8 @@ "issue_type": "ND001", "launch_name": "Launch 1", "log_level": 40000, - "original_message_lines": 1, - "original_message_words_number": 2, + "message_lines": 1, + "message_words_number": 2, "message": "Message http : localhost/admin \n java.lang.NoClassDefFoundError\r", "test_item": 3, "start_time": "2020-01-15 10:57:43", diff --git a/test_res/fixtures/two_hits_search_rs.json b/test_res/fixtures/two_hits_search_rs.json index 27dc9354..dc914666 100644 --- a/test_res/fixtures/two_hits_search_rs.json +++ b/test_res/fixtures/two_hits_search_rs.json @@ -22,8 +22,8 @@ "issue_type": "AB001", "launch_name": "Launch 1", "log_level": 40000, - "original_message_lines": 1, - "original_message_words_number": 2, + "message_lines": 1, + "message_words_number": 2, "message": "Message SPECIALNUMBER http : localhost/admin \n java.lang.NoClassDefFoundError\r", "test_item": 1, "start_time": "2020-01-15 10:57:43", @@ -59,8 +59,8 @@ "issue_type": "PB001", "launch_name": "Launch 1", "log_level": 40000, - "original_message_lines": 1, - "original_message_words_number": 2, + "message_lines": 1, + "message_words_number": 2, "message": "Message", "test_item": 2, "unique_id": "unique3", diff --git a/test_res/fixtures/two_hits_search_rs_explained.json b/test_res/fixtures/two_hits_search_rs_explained.json index 0a33e5aa..b591e910 100644 --- a/test_res/fixtures/two_hits_search_rs_explained.json +++ b/test_res/fixtures/two_hits_search_rs_explained.json @@ -24,8 +24,8 @@ "launch_name": "Launch test", "launch_id": 12, "log_level": 40000, - "original_message_lines": 28, - "original_message_words_number": 50, + "message_lines": 28, + "message_words_number": 50, "message": "java.lang.NullPointerException status\n\tat tv.horizon.web.ta.ui.elements.assets.Promotion.setFeed(Promotion.java:)\n\tat tv.horizon.web.ta.ui.elements.assets.Promotion.(Promotion.java:)\n\tat tv.horizon.web.ta.services.navigation.AssetsManager.checkTitleCardForPromotions(AssetsManager.java:)\n\tat tv.horizon.web.tests.checklist.AspotAndPromotionChecklist.checkPromotionFeeds(AspotAndPromotionChecklist.java:)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:)\n\tat java.lang.reflect.Method.invoke(Method.java:)\n\tat org.testng.internal.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:)\n\tat org.testng.internal.MethodInvocationHelper$.runTestMethod(MethodInvocationHelper.java:)\n\tat org.springframework.test.context.testng.AbstractTestNGSpringContextTests.run(AbstractTestNGSpringContextTests.java:)\n\tat org.testng.internal.MethodInvocationHelper.invokeHookable(MethodInvocationHelper.java:)\n\tat org.testng.internal.Invoker.invokeMethod(Invoker.java:)\n\tat org.testng.internal.Invoker.invokeTestMethod(Invoker.java:)\n\tat org.testng.internal.Invoker.invokeTestMethods(Invoker.java:)\n\tat org.testng.internal.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:)\n\tat org.testng.internal.TestMethodWorker.run(TestMethodWorker.java:)\n\tat org.testng.TestRunner.privateRun(TestRunner.java:)\n\tat org.testng.TestRunner.run(TestRunner.java:)\n\tat org.testng.SuiteRunner.runTest(SuiteRunner.java:)\n\tat org.testng.SuiteRunner.access$(SuiteRunner.java:)\n\tat org.testng.SuiteRunner$SuiteWorker.run(SuiteRunner.java:)\n\tat org.testng.internal.thread.ThreadUtil$.call(ThreadUtil.java:)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:)\n\tat java.lang.Thread.run(Thread.java:)\n", "test_item": 1, "test_item_name": "first test 1", @@ -63,8 +63,8 @@ "launch_name": "Launch 1", "launch_id": 13, "log_level": 40000, - "original_message_lines": 29, - "original_message_words_number": 51, + "message_lines": 29, + "message_words_number": 51, "message": "java.lang.NullPointerException status\n\tat tv.horizon.web.ta.ui.elements.assets.Promotion.setFeed(Promotion.java:)\n\tat tv.horizon.web.ta.ui.elements.assets.Promotion.(Promotion.java:)\n\tat tv.horizon.web.ta.ui.elements.aspot.Aspot.isMostWatchItemShouldBeDisplayed(Aspot.java:)\n\tat tv.horizon.web.tests.checklist.AspotAndPromotionChecklist.checkAspot(AspotAndPromotionChecklist.java:)\n\tat tv.horizon.web.tests.checklist.AspotAndPromotionChecklist.calculate(AspotAndPromotionChecklist.java:)\n\tat tv.horizon.web.tests.checklist.AspotAndPromotionChecklist.evaluate(AspotAndPromotionChecklist.java:)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:)\n\tat java.lang.reflect.Method.invoke(Method.java:)\n\tat org.testng.internal.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:)\n\tat org.testng.internal.MethodInvocationHelper$.runTestMethod(MethodInvocationHelper.java:)\n\tat org.springframework.test.context.testng.AbstractTestNGSpringContextTests.run(AbstractTestNGSpringContextTests.java:)\n\tat org.testng.internal.MethodInvocationHelper.invokeHookable(MethodInvocationHelper.java:)\n\tat org.testng.internal.Invoker.invokeMethod(Invoker.java:)\n\tat org.testng.internal.Invoker.invokeTestMethod(Invoker.java:)\n\tat org.testng.internal.Invoker.invokeTestMethods(Invoker.java:)\n\tat org.testng.internal.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:)\n\tat org.testng.internal.TestMethodWorker.run(TestMethodWorker.java:)\n\tat org.testng.TestRunner.privateRun(TestRunner.java:)\n\tat org.testng.TestRunner.run(TestRunner.java:)\n\tat org.testng.SuiteRunner.runTest(SuiteRunner.java:)\n\tat org.testng.SuiteRunner.access$(SuiteRunner.java:)\n\tat org.testng.SuiteRunner$SuiteWorker.run(SuiteRunner.java:)\n\tat org.testng.internal.thread.ThreadUtil$.call(ThreadUtil.java:)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:)\n\tat java.lang.Thread.run(Thread.java:)\n", "test_item": 2, "test_item_name": "first test 2", diff --git a/test_res/fixtures/two_hits_search_rs_second_message.json b/test_res/fixtures/two_hits_search_rs_second_message.json index 34097cec..e70fdc8f 100644 --- a/test_res/fixtures/two_hits_search_rs_second_message.json +++ b/test_res/fixtures/two_hits_search_rs_second_message.json @@ -22,8 +22,8 @@ "issue_type": "AB001", "launch_name": "Launch 1", "log_level": 40000, - "original_message_lines": 1, - "original_message_words_number": 2, + "message_lines": 1, + "message_words_number": 2, "message": "Message \n Message \n Message 'prod_en' /src/prod/results.html \n java.lang.NoClassDefFoundError\n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator", "test_item": 1, "start_time": "2020-01-15 10:57:43", @@ -59,8 +59,8 @@ "issue_type": "PB001", "launch_name": "Launch 1", "log_level": 40000, - "original_message_lines": 1, - "original_message_words_number": 2, + "message_lines": 1, + "message_words_number": 2, "message": "Message", "test_item": 2, "unique_id": "unique3", diff --git a/test_res/fixtures/two_hits_search_rs_small_logs.json b/test_res/fixtures/two_hits_search_rs_small_logs.json index 04fdfdc4..ccb82e26 100644 --- a/test_res/fixtures/two_hits_search_rs_small_logs.json +++ b/test_res/fixtures/two_hits_search_rs_small_logs.json @@ -24,8 +24,8 @@ "launch_name": "Launch 1", "launch_id": 13, "log_level": 40000, - "original_message_lines": 2, - "original_message_words_number": 6, + "message_lines": 2, + "message_words_number": 6, "message": "", "test_item": 1, "test_item_name": "first test", @@ -63,8 +63,8 @@ "launch_name": "Launch 1", "launch_id": 13, "log_level": 40000, - "original_message_lines": 2, - "original_message_words_number": 7, + "message_lines": 2, + "message_words_number": 7, "message": "", "test_item": 2, "test_item_name": "first test 2", diff --git a/test_res/fixtures/two_hits_search_with_big_messages_rs.json b/test_res/fixtures/two_hits_search_with_big_messages_rs.json index 8a370dd6..ff77f967 100644 --- a/test_res/fixtures/two_hits_search_with_big_messages_rs.json +++ b/test_res/fixtures/two_hits_search_with_big_messages_rs.json @@ -24,8 +24,8 @@ "launch_name": "Launch 1", "log_level": 40000, "message": "Message AB \n Message AB \n Message AB", - "original_message_lines": 3, - "original_message_words_number": 2, + "message_lines": 3, + "message_words_number": 2, "test_item": 1, "unique_id": "unique1", "test_case_hash": -1126886180, @@ -43,8 +43,8 @@ "launch_name": "Launch 1", "log_level": 40000, "message": "Message PB \n Message PB \n Message PB", - "original_message_lines": 3, - "original_message_words_number": 2, + "message_lines": 3, + "message_words_number": 2, "test_item": 1, "unique_id": "unique1", "test_case_hash": -1126886180, diff --git a/test_res/fixtures/two_hits_with_no_defect.json b/test_res/fixtures/two_hits_with_no_defect.json index 979ae485..0936f5b7 100644 --- a/test_res/fixtures/two_hits_with_no_defect.json +++ b/test_res/fixtures/two_hits_with_no_defect.json @@ -22,8 +22,8 @@ "issue_type": "ND001", "launch_name": "Launch 1", "log_level": 40000, - "original_message_lines": 1, - "original_message_words_number": 2, + "message_lines": 1, + "message_words_number": 2, "message": "Message http : localhost/admin \n java.lang.NoClassDefFoundError\r", "test_item": 34, "start_time": "2020-01-15 10:57:43", @@ -58,8 +58,8 @@ "issue_type": "PB001", "launch_name": "Launch 1", "log_level": 40000, - "original_message_lines": 1, - "original_message_words_number": 2, + "message_lines": 1, + "message_words_number": 2, "message": "Message http : localhost/admin \n java.lang.NoClassDefFoundError\r", "test_item": 23, "start_time": "2020-01-15 10:57:43",