Literature_Review_Tool_v.1.0.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e09c1058",
   "metadata": {},
   "source": [
    "# 安装依赖部分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ebc384a",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install langchain\n",
    "!pip install unstructured # The unstructured library provides open-source components for pre-processing text documents such as PDFs, HTML and Word Documents. \n",
    "!pip install openai\n",
    "!pip install pybind11 # pybind11 is a lightweight header-only library that exposes C++ types in Python\n",
    "!pip install chromadb # the AI-native open-source embedding database\n",
    "!pip install Cython # Cython is an optimising static compiler for both the Python programming language\n",
    "!pip3 install \"git+https://github.com/philferriere/cocoapi.git#egg=pycocotools&subdirectory=PythonAPI\" # COCO is a large image dataset designed for object detection, segmentation, person keypoints detection, stuff segmentation, and caption generation\n",
    "!pip install unstructured[local-inference]\n",
    "!CC=clang CXX=clang++ ARCHFLAGS=\"-arch x86_64\" pip install 'git+https://github.com/facebookresearch/detectron2.git' # Detectron2 is Facebook AI Research's next generation library that provides state-of-the-art detection and segmentation algorithms.\n",
    "!pip install layoutparser[layoutmodels,tesseract] # A Unified Toolkit for Deep Learning Based Document Image Analysis\n",
    "!pip install pytesseract # Python-tesseract is an optical character recognition (OCR) tool for python.\n",
    "!pip install Pillow==9.0.0 # The Python Imaging Library adds image processing capabilities to your Python interpreter. \n",
    "!pip install tiktoken\n",
    "!apt-get install poppler-utils\n",
    "!sudo apt-get install tesseract-ocr\n",
    "!sudo apt-get install tesseract-ocr-math\n",
    "!pip install habanero\n",
    "!pip install PyPDF2\n",
    "!pip install bibtexparser\n",
    "!pip install pymupdf\n",
    "!pip install kor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "69b5c3e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')\n",
    "!cd /content/drive/MyDrive/Literature_Review_tool"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fce0dd99",
   "metadata": {},
   "source": [
    "# 提取文本部分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "61d2d5b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_pdf(path,openai_api_key,chunk_size,chunk_overlap):\n",
    "  from langchain.document_loaders import PyMuPDFLoader,DirectoryLoader,UnstructuredPDFLoader\n",
    "  from detectron2.config import get_cfg\n",
    "  from PyPDF2 import PdfReader\n",
    "\n",
    "  cfg = get_cfg()    \n",
    "  cfg.MODEL.DEVICE = 'gpu'\n",
    "\n",
    "  import os\n",
    "\n",
    "  file_names = os.listdir(path)\n",
    "  pdf_file_names = [path + '/'+file_name for file_name in file_names if file_name.endswith('.pdf')]\n",
    "\n",
    "  docs =[]\n",
    "\n",
    "  import re\n",
    "\n",
    "  for pdf in pdf_file_names:\n",
    "    source = extract_doi(pdf)\n",
    "\n",
    "    if source != 'None':\n",
    "      doc = PyMuPDFLoader(pdf).load()\n",
    "      for element in doc:\n",
    "        element.metadata = source\n",
    "        element.page_content = re.sub('\\n+',' ',element.page_content.strip())\n",
    "        docs.append(element)\n",
    "  \n",
    "    else:\n",
    "      doc = PyMuPDFLoader(pdf).load()\n",
    "      print(f\"{pdf} is not identified! Using other strategy!!\")\n",
    "      source = extract_doi_llm(doc,openai_api_key)\n",
    "      if source != 'None':\n",
    "        for element in doc:\n",
    "          element.metadata = source\n",
    "      for element in doc:\n",
    "        element.page_content = re.sub('\\n+',' ',element.page_content.strip())\n",
    "        docs.append(element)\n",
    "  \n",
    "  \n",
    "  \n",
    "  from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "\n",
    "  text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size,chunk_overlap = chunk_overlap)\n",
    "\n",
    "  split_docs = text_splitter.split_documents(docs)\n",
    "\n",
    "  return split_docs\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7728df14",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_info(path):\n",
    "    from PyPDF2 import PdfReader\n",
    "    with open(path, 'rb') as f:\n",
    "        pdf = PdfReader(f)\n",
    "        info = pdf.metadata\n",
    "        return info\n",
    "    \n",
    "def extract_doi(path):\n",
    "\n",
    "    source = 0\n",
    "    info = get_info(path)\n",
    "    if '/doi' in info:\n",
    "        doi = info['/doi']\n",
    "    elif '/Subject' in info:\n",
    "        Subject = info['/Subject']\n",
    "        if 'doi:' in Subject:\n",
    "            Subject = Subject.split('doi:')\n",
    "            doi = Subject[1]\n",
    "        else:\n",
    "          source = 'None'\n",
    "    elif '/WPS-ARTICLEDOI' in info:\n",
    "        doi = info['/WPS-ARTICLEDOI']\n",
    "    else:\n",
    "        source = 'None'\n",
    "    \n",
    "    if source != 'None':\n",
    "        import habanero\n",
    "        import time\n",
    "        citation = habanero.cn.content_negotiation(ids = doi,format='bibentry')\n",
    "        time.sleep(5)\n",
    "        import bibtexparser\n",
    "        citation = bibtexparser.loads(citation)\n",
    "        citation = citation.entries[0]\n",
    "        source = {'author':citation['author'],\n",
    "              'year':citation['year'],\n",
    "              'title':citation['title'],\n",
    "              'journal':citation['journal'],\n",
    "              }\n",
    "    \n",
    "    return source"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f76cbbf6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_doi_llm(doc,openai_api_key):\n",
    "\n",
    "  import re \n",
    "\n",
    "  doc[0].page_content = re.sub('\\n+',' ',doc[0].page_content.strip())\n",
    "\n",
    "  from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "  text_splitter = RecursiveCharacterTextSplitter(chunk_size=1500,chunk_overlap = 50)\n",
    "  split_docs = text_splitter.split_documents(doc)\n",
    "  abstract = split_docs[0]\n",
    "  doi = extract_chain(abstract,openai_api_key)\n",
    "\n",
    "  if doi != 'None':\n",
    "    import habanero\n",
    "    import time\n",
    "    citation = habanero.cn.content_negotiation(ids = doi,format='bibentry')\n",
    "    time.sleep(5)\n",
    "    import bibtexparser\n",
    "    citation = bibtexparser.loads(citation)\n",
    "    citation = citation.entries[0]\n",
    "    source = {'author':citation['author'],\n",
    "            'year':citation['year'],\n",
    "            'title':citation['title'],\n",
    "            'journal':citation['journal'],\n",
    "            }\n",
    "    return source\n",
    "  else:\n",
    "    source = 'None'\n",
    "    return source\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "08271147",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_chain(abstract,openai_api_key):\n",
    "  from kor.extraction import create_extraction_chain\n",
    "  from kor.nodes import Object, Text, Number\n",
    "  from langchain.chat_models import ChatOpenAI\n",
    "\n",
    "  llm = ChatOpenAI(\n",
    "    model_name=\"gpt-3.5-turbo\",\n",
    "    openai_api_key = openai_api_key,\n",
    "    temperature=0,\n",
    "    )\n",
    "  schema = Object(\n",
    "      id=\"doi\",\n",
    "      description=\"doi is a digital identifier.It typically starts with 10. followed by a numeric prefix, such as 10.1000/182.\",\n",
    "      attributes=[\n",
    "          Text(\n",
    "              id=\"doi\",\n",
    "              description='doi is a digital identifier. It typically starts with \"10.\" followed by a numeric prefix, such as 10.1000/182.',\n",
    "              examples=[\n",
    "                  ('American Economic Journal: Economic Policy 2015, 7(4): 223–242  http://dx.doi.org/10.1257/pol.20130367 223 Water Pollution Progress at Borders: The','http://dx.doi.org/10.1257/pol.20130367'),\n",
    "                  ('Environment and Development Economics (2020), 1–17 doi:10.1017/S1355770X2000025X EDE RESEARCH ARTICLE Political incentives, Party Congress, and pollution cycle: empirical evidence from China Zhihua Tian,1 and Yanfang Tian2* 1School of Economics, Zhejiang University of Technology, Hangzhou','10.1017/S1355770X2000025X')\n",
    "                  ],\n",
    "               many=True\n",
    "               )\n",
    "          ],\n",
    "          many=False\n",
    "          )\n",
    "  chain = create_extraction_chain(llm, schema, encoder_or_encoder_class='json')\n",
    "  output = chain.predict_and_parse(text=abstract.page_content)\n",
    "  if 'doi' not in output['data']:\n",
    "    print(f\"LLM strategy failed!!{abstract.metadata['source']} Please manually add it!!\")\n",
    "    source = 'None'\n",
    "    \n",
    "    return source\n",
    "    \n",
    "  else:\n",
    "    doi = output['data']['doi']['doi'][0]\n",
    "    if 'doi=' in doi:\n",
    "      doi = doi.split('doi=')[1]\n",
    "    return doi\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4e6ea70",
   "metadata": {},
   "source": [
    "# 生成数据库部分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "074881a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_or_load_vectorstore(split_docs,device,model_name,generate,persist_directory,collection_name):\n",
    "    \n",
    "    from langchain.vectorstores import Chroma\n",
    "    from langchain.embeddings import HuggingFaceEmbeddings\n",
    "\n",
    "    \n",
    "    model_kwargs = {'device':device}\n",
    "    model_name = model_name\n",
    "    embeddings = HuggingFaceEmbeddings(model_name=model_name, model_kwargs=model_kwargs)\n",
    "    \n",
    "    persist_directory = persist_directory\n",
    "    collection_name = collection_name\n",
    "    \n",
    "    if generate == True:\n",
    "        vectorstore = Chroma.from_documents(split_docs,embeddings,collection_name=collection_name,persist_directory = persist_directory)\n",
    "        vectorstore.persist()\n",
    "        \n",
    "    else:\n",
    "        vectordb = Chroma(collection_name=collection_name, persist_directory=persist_directory, embedding_function=embeddings)\n",
    "        \n",
    "    return vectordb"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f864482b",
   "metadata": {},
   "source": [
    "# Chain生成并匹配"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "cd59c471",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_chain_output(query,vectordb,k,openai_api_key):\n",
    "    \n",
    "    docs = vectordb.similarity_search(query,5,include_metadata=True)\n",
    "    \n",
    "    from langchain.chat_models import ChatOpenAI\n",
    "    \n",
    "    llm = ChatOpenAI(openai_api_key=openai_api_key, temperature=0, model_name=\"gpt-3.5-turbo\")\n",
    "    \n",
    "    from langchain.prompts import PromptTemplate,ChatPromptTemplate,HumanMessagePromptTemplate\n",
    "    from langchain.llms import OpenAI\n",
    "    \n",
    "    from langchain.output_parsers import PydanticOutputParser\n",
    "    from pydantic import BaseModel, Field, validator\n",
    "    from typing import List,Union,Optional\n",
    "    \n",
    "    class Sentence(BaseModel):\n",
    "        sentence: List[str] = Field(description=\"The sentence in the given document which is the most similar to the query provided\")\n",
    "        source: List[str] = Field(description=\"The meta source of the paper\")\n",
    "        score: List[float] = Field(description = \"The similarity score between the sentence selected and the query provided\")\n",
    "            \n",
    "    parser = PydanticOutputParser(pydantic_object=Sentence)\n",
    "    \n",
    "    question_template = \"\"\"\n",
    "    Given the document and query, find three sentences in the document that are most similar in meaning to the query. \n",
    "    Return the sentences, the meta source of the sentences and the cosine similarity scores. \n",
    "    If no similar sentences is found, return the sentence with highest cosine siliarity scores.\n",
    "    {query}\n",
    "    ===========\n",
    "    {context}\n",
    "    ===========\n",
    "    {format_instructions}\n",
    "    \n",
    "    \"\"\"\n",
    "\n",
    "    from langchain.chains.question_answering import load_qa_chain\n",
    "    from langchain import LLMChain\n",
    "\n",
    "    PROMPT = PromptTemplate(template = question_template,\n",
    "                            input_variables=['query','context'],\n",
    "                            partial_variables = {\"format_instructions\":parser.get_format_instructions()})\n",
    "    \n",
    "    llm_chain = LLMChain(llm=llm,prompt = PROMPT)\n",
    "    \n",
    "    output = llm_chain({\"query\":query,\"context\":docs})\n",
    "    \n",
    "    return output,docs\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "7c3722ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_text_match(output,query,docs):\n",
    "    \n",
    "    import re\n",
    "    text = re.sub(\"\\n+\",\"\",output['text'])\n",
    "    \n",
    "    import json\n",
    "    json_obj = json.loads(text)\n",
    "    \n",
    "    \n",
    "    if \"properties\" in json_obj:\n",
    "        print('No result was found, Using embedding searching strategy!!!')\n",
    "        split_docs = split_for_embedding(docs)\n",
    "        similar_sentence = search_cosine_similarity(query,split_docs,embeddings)\n",
    "    \n",
    "        for i,element in enumerate(similar_sentence):\n",
    "            print(f'The {i} sentence')\n",
    "            print(f\"Sentence:{element['sentences']}\")\n",
    "            print(f\"Source:{element['source']}\")\n",
    "            print(f\"Score:{element['score']}\")\n",
    "            print(\"========\")\n",
    "            print(\"========\")\n",
    "    else:\n",
    "        for i in range(3):\n",
    "            print(f'The {i} sentence')\n",
    "            print(f\"Sentence:{json_obj['sentence'][i]}\")\n",
    "            print(f\"Source:{json_obj['source'][i]}\")\n",
    "            print(f\"Score:{json_obj['score'][i]}\")\n",
    "            print(\"========\")\n",
    "            print(\"========\")\n",
    "\n",
    "def split_for_embedding(docs): ##输入docs(list),输出split_for embedding(list)\n",
    "    for_embedding = []\n",
    "    for content in docs:\n",
    "        new_content = content.page_content.replace('et al.','et al。')\n",
    "        new_content = new_content.split('.')\n",
    "        \n",
    "        meta_data = content.metadata['source']\n",
    "        \n",
    "        for split_content in new_content:\n",
    "            split_content = split_content.replace('。','.')\n",
    "            \n",
    "            if len(split_content) < 30:\n",
    "                continue\n",
    "            else:\n",
    "                for_embedding.append({\"content\":split_content,\"source\":meta_data})\n",
    "                \n",
    "    return for_embedding\n",
    "\n",
    "def search_cosine_similarity(query,split_docs,embeddings):  ##query-str,split_docs-list,embeddings-embeddings()\n",
    "    split_docs_content = [content['content'] for content in split_docs]\n",
    "    embed_docs = embeddings.embed_documents(split_docs_content)\n",
    "    embed_query= embeddings.embed_query(query)\n",
    "    \n",
    "    from openai.embeddings_utils import cosine_similarity\n",
    "    \n",
    "    cos_index = []\n",
    "    for embed_doc in embed_docs:\n",
    "        cos_index.append(cosine_similarity(embed_doc,embed_query))\n",
    "    \n",
    "    #这边是根据大小建立索引\n",
    "    idx = sorted(range(len(cos_index)),key=lambda k:cos_index[k]) #根据cos_index的大小进行排序\n",
    "    final_similar_list = []\n",
    "    for index in idx[-3:]:\n",
    "        unit = {}\n",
    "        unit['sentences']=split_docs_content[index]\n",
    "        unit['source']=split_docs[index]['source']\n",
    "        unit['score']=cos_index[index]\n",
    "        final_similar_list.append(unit)\n",
    "    \n",
    "    return final_similar_list\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "50e1be47",
   "metadata": {},
   "source": [
    "# 主程序"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "74773c01",
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'detectron2'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[8], line 8\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtorch\u001b[39;00m \n\u001b[0;32m      6\u001b[0m device \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcuda\u001b[39m\u001b[38;5;124m'\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mcuda\u001b[38;5;241m.\u001b[39mis_available() \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcpu\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m----> 8\u001b[0m split_docs \u001b[38;5;241m=\u001b[39m \u001b[43mload_pdf\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpath\u001b[49m\u001b[43m,\u001b[49m\u001b[43mopenai_api_key\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mopenai_api_key\u001b[49m\u001b[43m,\u001b[49m\u001b[43mchunk_size\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m1500\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mchunk_overlap\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m100\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[0;32m      9\u001b[0m vectordb \u001b[38;5;241m=\u001b[39m generate_or_load_vectorstore(split_docs,device,model_name \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msentence-transformers/all-mpnet-base-v2\u001b[39m\u001b[38;5;124m\"\u001b[39m,generate\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[0;32m     10\u001b[0m                                         persist_directory \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mhuggingface_index\u001b[39m\u001b[38;5;124m'\u001b[39m,collection_name\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mpaper_index\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[0;32m     11\u001b[0m                                         )\n",
      "Cell \u001b[1;32mIn[2], line 3\u001b[0m, in \u001b[0;36mload_pdf\u001b[1;34m(path, openai_api_key, chunk_size, chunk_overlap)\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mload_pdf\u001b[39m(path,openai_api_key,chunk_size,chunk_overlap):\n\u001b[0;32m      2\u001b[0m   \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mlangchain\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdocument_loaders\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m PyMuPDFLoader,DirectoryLoader,UnstructuredPDFLoader\n\u001b[1;32m----> 3\u001b[0m   \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mdetectron2\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mconfig\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m get_cfg\n\u001b[0;32m      4\u001b[0m   \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mPyPDF2\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m PdfReader\n\u001b[0;32m      6\u001b[0m   cfg \u001b[38;5;241m=\u001b[39m get_cfg()    \n",
      "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'detectron2'"
     ]
    }
   ],
   "source": [
    "path = \"/content/drive/MyDrive/Literature_Review_tool/literature\"\n",
    "os.environ['openai_api_key'] = 'my_value'\n",
    "openai_api_key = os.environ.get('openai_api_key')\n",
    "\n",
    "import torch \n",
    "\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "\n",
    "split_docs = load_pdf(path=path,openai_api_key=openai_api_key,chunk_size =1500, chunk_overlap=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "e86429aa",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using embedded DuckDB with persistence: data will be stored in: huggingface_index\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The 0 sentence\n",
      "Sentence:The Central Organi- zation Department emerged as the key human resource manager of the CCP, although senior-level promotions were ultimately decided by members of the Politburo (Nathan and Gilley 2002).\n",
      "Source:American Political Science Review\n",
      "Score:0.707\n",
      "========\n",
      "========\n",
      "The 1 sentence\n",
      "Sentence:After reform began in 1978, education creden- tials came to play a prominent role in the advancement of lower level cadres, and older cadres were strongly encouraged to retire (Cui 2003; Landry 2008; Manion 1993; Walder, Li, and Treiman 2000).\n",
      "Source:American Political Science Review\n",
      "Score:0.707\n",
      "========\n",
      "========\n",
      "The 2 sentence\n",
      "Sentence:Finally, when a centralized ﬁscal system was ﬁrst implemented in the mid-1990s, the CCP used the cadre evaluation system to ensure that provincial lead- ers cooperated with central tax ofﬁcials in maximizing revenue for the central government.\n",
      "Source:American Political Science Review\n",
      "Score:0.707\n",
      "========\n",
      "========\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "vectordb = generate_or_load_vectorstore(split_docs=True,device=device,model_name = \"sentence-transformers/all-mpnet-base-v2\",generate=False,persist_directory = 'huggingface_index',collection_name='paper_index')\n",
    "\n",
    "query = \"However, the China's cadre evaluation system is different from the voting system in the Western countries\"\n",
    "\n",
    "output,docs = get_chain_output(query,vectordb,k=5,openai_api_key=openai_api_key)\n",
    "\n",
    "final_list = run_text_match(output,query,docs)\n",
    "\n",
    "print(final_list)\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a87aa835",
   "metadata": {},
   "outputs": [],
   "source": [
    "'huggingface_index',collection_name='paper_index'\n",
    "The political business cycle is caused by opportunism"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "langchain",
   "language": "python",
   "name": "langchain"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}