Fixed chunking pipeline, query was not generating due to mismatchd ve…

…ctor fields, like name etc, updated knn index details and chunking details Signed-off-by: hmumtazz <[email protected]>
opensearch-project · Dec 5, 2024 · d877bca · d877bca
1 parent c02fe1b
commit d877bca
Show file tree

Hide file tree

Showing 5 changed files with 48 additions and 12 deletions.
diff --git a/config.ini b/config.ini
@@ -0,0 +1,26 @@
+[DEFAULT]
+service_type = managed
+region = us-west-2
+iam_principal = arn:aws:iam::615299771255:user/hmumtazz
+collection_name = 
+opensearch_endpoint = https://search-hashim-test5-eivrlyacr3n653fnkkrg2yab7u.aos.us-west-2.on.aws
+opensearch_username = admin
+opensearch_password = MyPassword123!
+default_search_method = semantic
+llm_model_id = amazon.titan-text-express-v1
+llm_max_token_count = 1000
+llm_temperature = 0.7
+llm_top_p = 0.9
+llm_stop_sequences = 
+ingest_pipeline_name = text-chunking-ingest-pipeline
+index_name = nemaz
+embedding_dimension = 1536
+space_type = l2
+ef_construction = 512
+number_of_shards = 1
+number_of_replicas = 2
+passage_text_field = passage_text
+passage_chunk_field = passage_chunk
+embedding_field = passage_embedding
+embedding_model_id = ecFOjZMBEYoaB6B-K7-l
+
diff --git a/opensearch_py_ml/ml_commons/rag_pipeline/rag/ingest.py b/opensearch_py_ml/ml_commons/rag_pipeline/rag/ingest.py
@@ -8,6 +8,7 @@
 import os
 import glob
 import json
+import tiktoken
 from tqdm import tqdm
 from colorama import Fore, Style, init
 from typing import List, Dict
@@ -201,10 +202,8 @@ def create_ingest_pipeline(self, pipeline_id: str):
                     {
                         "text_chunking": {
                             "algorithm": {
-                                "fixed_token_length": {
-                                    "token_limit": token_limit,
-                                    "overlap_rate": overlap_rate,
-                                    "tokenizer": tokenizer
+                                "delimiter": {
+                                    "delimiter": "."
                                 }
                             },
                             "field_map": {

diff --git a/opensearch_py_ml/ml_commons/rag_pipeline/rag/opensearch_connector.py b/opensearch_py_ml/ml_commons/rag_pipeline/rag/opensearch_connector.py
@@ -262,18 +262,28 @@ def search_by_vector(self, vector: list, k: int = 5) -> list:
         :param k: The number of top results to retrieve.
         :return: A list of search hits.
         """
+        # Retrieve field names from the config
+        embedding_field = self.config.get('embedding_field', 'passage_embedding')
+        passage_text_field = self.config.get('passage_text_field', 'passage_text')
+        passage_chunk_field = self.config.get('passage_chunk_field', 'passage_chunk')
+
         try:
-            # Execute the KNN search query
+            # Execute the KNN search query using the correct field name
             response = self.opensearch_client.search(
                 index=self.index_name,
                 body={
                     "size": k,
-                    "_source": ["nominee_text", "passage_chunk"],
+                    "_source": [passage_text_field, passage_chunk_field],
                     "query": {
-                        "knn": {
-                            "nominee_vector": {
-                                "vector": vector,
-                                "k": k
+                        "nested": {
+                            "path": embedding_field,
+                            "query": {
+                                "knn": {
+                                    f"{embedding_field}.knn": {
+                                        "vector": vector,
+                                        "k": k
+                                    }
+                                }
                             }
                         }
                     }

diff --git a/opensearch_py_ml/ml_commons/rag_pipeline/rag/query.py b/opensearch_py_ml/ml_commons/rag_pipeline/rag/query.py
@@ -211,7 +211,9 @@ def bulk_query_semantic(self, queries: List[str], k: int = 5) -> List[dict]:
                 # Perform vector-based search
                 hits = self.opensearch.search_by_vector(vector, k)
                 # Concatenate the retrieved passages as context
-                context = '\n'.join([hit['_source']['nominee_text'] for hit in hits])
+                context = '\n'.join(
+                    [chunk for hit in hits for chunk in hit['_source'].get('passage_chunk', [])]
+                )
                 results.append({
                     'query': queries[i],
                     'context': context,

diff --git a/opensearch_py_ml/ml_commons/rag_pipeline/rag/rag_setup.py b/opensearch_py_ml/ml_commons/rag_pipeline/rag/rag_setup.py
@@ -19,7 +19,6 @@
 from urllib.parse import urlparse
 from opensearchpy import OpenSearch, RequestsHttpConnection, AWSV4SignerAuth
 from colorama import Fore, Style, init
-import ssl
 
 from opensearch_py_ml.ml_commons.rag_pipeline.rag.opensearch_connector import OpenSearchConnector
 from opensearch_py_ml.ml_commons.rag_pipeline.rag.serverless import Serverless