you_pinecone_multiretriever.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "ohp-N8APT5YR"
      },
      "outputs": [],
      "source": [
        "!pip install -qU \\\n",
        "  langchain==0.0.335 \\\n",
        "  pinecone-client==2.2.4 \\\n",
        "  openai \\\n",
        "  datasets \\\n",
        "  tiktoken\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "6_1V2yd2RVH1"
      },
      "outputs": [],
      "source": [
        "from langchain.retrievers.you import YouRetriever\n",
        "from langchain.chains import RetrievalQA\n",
        "from langchain.chat_models.openai import ChatOpenAI\n",
        "import os\n",
        "\n",
        "os.environ[\"OPENAI_API_KEY\"] = \"<<OPENAI_API_KEY>>\"\n",
        "os.environ[\"YDC_API_KEY\"] = \"<<YDC_API_KEY>>\"\n",
        "\n",
        "llm = ChatOpenAI(model=\"gpt-3.5-turbo-16k\")\n",
        "\n",
        "you_retriever = YouRetriever()\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-3kKs-5_Zvw-"
      },
      "source": [
        "## Create Pinecone Indes"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4PIbLT-bWimO",
        "outputId": "5c319dc9-b6e1-4718-82f6-c36a2303949f"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Dataset({\n",
              "    features: ['doi', 'chunk-id', 'chunk', 'id', 'title', 'summary', 'source', 'authors', 'categories', 'comment', 'journal_ref', 'primary_category', 'published', 'updated', 'references'],\n",
              "    num_rows: 41584\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ],
      "source": [
        "from datasets import load_dataset\n",
        "\n",
        "dataset = load_dataset(\n",
        "    \"jamescalam/ai-arxiv-chunked\",\n",
        "    split=\"train\"\n",
        ")\n",
        "\n",
        "dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "NE3HGAnOZ1f6"
      },
      "outputs": [],
      "source": [
        "import pinecone\n",
        "\n",
        "# get API key from app.pinecone.io and environment from console\n",
        "pinecone.init(\n",
        "    api_key=\"<<PINECONE_API_KEY>>\",\n",
        "    environment=\"<<PINECONE_ENV>>\"\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "z_613JFoaVc8"
      },
      "outputs": [],
      "source": [
        "import time\n",
        "\n",
        "index_name = 'you-pinecone'\n",
        "\n",
        "if index_name not in pinecone.list_indexes():\n",
        "    pinecone.create_index(\n",
        "        index_name,\n",
        "        dimension=1536,\n",
        "        metric='cosine'\n",
        "    )\n",
        "    # wait for index to finish initialization\n",
        "    while not pinecone.describe_index(index_name).status['ready']:\n",
        "        time.sleep(1)\n",
        "\n",
        "index = pinecone.Index(index_name)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "2FwLb0PhaetT"
      },
      "outputs": [],
      "source": [
        "from langchain.embeddings.openai import OpenAIEmbeddings\n",
        "\n",
        "embed_model = OpenAIEmbeddings(\n",
        "    model=\"text-embedding-ada-002\",\n",
        "    disallowed_special=()\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49,
          "referenced_widgets": [
            "052489e2f8f44922a7c0921f586d9434",
            "0e29cd87f84d443789b1381c305f94eb",
            "50f29efa1f004ac99f70e36f0e584954",
            "d0194b6d5d9e4dd1943373e3f9dcc27b",
            "aa183e6299d747b6b3b9cc705de44909",
            "2dfa397456cb475d86aa842f848c9f3e",
            "6bd3fd3959e84eddaf03acb4cade42c4",
            "c7b10b88f2214064925d980b0466fb9d",
            "5c12481a504e43fd8a2ff24393a7c617",
            "852d273e2e5740fbb102e2a1c8dc1459",
            "d99c983ccb9f4c1f80d66d2400f660bb"
          ]
        },
        "id": "JgX-ot35ag31",
        "outputId": "44d6e33b-5ff3-40c0-d8ac-d1f19a38b608"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/416 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "052489e2f8f44922a7c0921f586d9434"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "from tqdm.auto import tqdm  # for progress bar\n",
        "\n",
        "data = dataset.to_pandas()  # this makes it easier to iterate over the dataset\n",
        "\n",
        "batch_size = 100\n",
        "\n",
        "for i in tqdm(range(0, len(data), batch_size)):\n",
        "    i_end = min(len(data), i+batch_size)\n",
        "    # get batch of data\n",
        "    batch = data.iloc[i:i_end]\n",
        "    # generate unique ids for each chunk\n",
        "    ids = [f\"{x['doi']}-{x['chunk-id']}\" for i, x in batch.iterrows()]\n",
        "    # get text to embed\n",
        "    texts = [x['chunk'] for _, x in batch.iterrows()]\n",
        "    # embed text\n",
        "    embeds = embed_model.embed_documents(texts)\n",
        "    # get metadata to store in Pinecone\n",
        "    metadata = [\n",
        "        {'text': x['chunk'],\n",
        "         'source': x['source'],\n",
        "         'title': x['title']} for i, x in batch.iterrows()\n",
        "    ]\n",
        "    # add to Pinecone\n",
        "    index.upsert(vectors=zip(ids, embeds, metadata))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mrNOhBiOak3D",
        "outputId": "87f494c0-44e2-4c45-dd24-3d12522c7eb1"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/langchain/vectorstores/pinecone.py:59: UserWarning: Passing in `embedding` as a Callable is deprecated. Please pass in an Embeddings object instead.\n",
            "  warnings.warn(\n"
          ]
        }
      ],
      "source": [
        "from langchain.vectorstores import Pinecone\n",
        "\n",
        "text_field = \"text\"  # the metadata field that contains our text\n",
        "\n",
        "# initialize the vector store object\n",
        "vectorstore = Pinecone(\n",
        "    index, embed_model.embed_query, text_field\n",
        ")\n",
        "pinecone_retriever = vectorstore.as_retriever()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "LJXhzDq4SuoH"
      },
      "outputs": [],
      "source": [
        "retriever_infos = [\n",
        "    {\n",
        "        \"name\": \"pinecone\",\n",
        "        \"description\": \"use this tool when you need information about LLMs (llama 2, gpt-4, etc) or ML\",\n",
        "        \"retriever\": pinecone_retriever\n",
        "    }, {\n",
        "        \"name\": \"you.com\",\n",
        "        \"description\": \"use this tool for general purpose queries that can be found on the web\",\n",
        "        \"retriever\": you_retriever\n",
        "    }\n",
        "]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "EBz0LpcqSwNB"
      },
      "outputs": [],
      "source": [
        "from langchain.chains.router.multi_retrieval_qa import MultiRetrievalQAChain\n",
        "\n",
        "retrieval_qa = MultiRetrievalQAChain.from_retrievers(\n",
        "    llm=llm, retriever_infos=retriever_infos,\n",
        "    verbose=True\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OR1t0Bxbbe7u",
        "outputId": "b53447f7-54f4-4246-b0a8-64fd7b50f339"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "\u001b[1m> Entering new MultiRetrievalQAChain chain...\u001b[0m\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/langchain/chains/llm.py:321: UserWarning: The predict_and_parse method is deprecated, instead pass an output parser directly to LLMChain.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "pinecone: {'query': 'tell me about the llama 2 LLM'}\n",
            "\u001b[1m> Finished chain.\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "res = retrieval_qa.invoke({\"input\": \"tell me about the llama 2 llm\"})"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "res"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "C58Nx5RZ-Dcl",
        "outputId": "3bff5482-a1b5-4da0-97e5-06cbb5dd8c88"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'input': 'tell me about the llama 2 llm',\n",
              " 'query': 'tell me about the llama 2 LLM',\n",
              " 'result': 'Llama 2 is a collection of pretrained and fine-tuned large language models (LLMs). These LLMs range in scale from 7 billion to 70 billion parameters. The main focus of Llama 2 is on optimizing the LLMs for dialogue use cases. \\n\\nAccording to the developers, Llama 2 models outperform open-source chat models on most benchmarks that were tested. They also claim that based on their humane evaluations for helpfulness and safety, Llama 2 models may be a suitable substitute for closed-source models.\\n\\nThe approach to fine-tuning and safety of the Llama 2 models is described in detail in the research work. However, the specific details of the fine-tuning and safety methods are not mentioned in the given context.\\n\\nOverall, Llama 2 aims to provide high-performing language models for dialogue applications, offering improved performance compared to existing open-source models.'}"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "A2egZBwcbuBI",
        "outputId": "41c73e4e-08b3-4282-fa23-8c87994af857"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "\u001b[1m> Entering new MultiRetrievalQAChain chain...\u001b[0m\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/langchain/chains/llm.py:321: UserWarning: The predict_and_parse method is deprecated, instead pass an output parser directly to LLMChain.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "you.com: {'query': 'who is the current German chancellor?'}\n",
            "\u001b[1m> Finished chain.\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "res = retrieval_qa.invoke({\"input\": \"who is the German chancellor now?\"})"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "n_qzKbCHbwuY",
        "outputId": "c64c6835-9c99-4faf-95df-27f34c10a96d"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'input': 'who is the German chancellor now?',\n",
              " 'query': 'who is the current German chancellor?',\n",
              " 'result': 'The current German chancellor is Olaf Scholz.'}"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ],
      "source": [
        "res"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "utIgAtb8cuzy"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "052489e2f8f44922a7c0921f586d9434": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_0e29cd87f84d443789b1381c305f94eb",
              "IPY_MODEL_50f29efa1f004ac99f70e36f0e584954",
              "IPY_MODEL_d0194b6d5d9e4dd1943373e3f9dcc27b"
            ],
            "layout": "IPY_MODEL_aa183e6299d747b6b3b9cc705de44909"
          }
        },
        "0e29cd87f84d443789b1381c305f94eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2dfa397456cb475d86aa842f848c9f3e",
            "placeholder": "​",
            "style": "IPY_MODEL_6bd3fd3959e84eddaf03acb4cade42c4",
            "value": "100%"
          }
        },
        "50f29efa1f004ac99f70e36f0e584954": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c7b10b88f2214064925d980b0466fb9d",
            "max": 416,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_5c12481a504e43fd8a2ff24393a7c617",
            "value": 416
          }
        },
        "d0194b6d5d9e4dd1943373e3f9dcc27b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_852d273e2e5740fbb102e2a1c8dc1459",
            "placeholder": "​",
            "style": "IPY_MODEL_d99c983ccb9f4c1f80d66d2400f660bb",
            "value": " 416/416 [26:51&lt;00:00,  3.60s/it]"
          }
        },
        "aa183e6299d747b6b3b9cc705de44909": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2dfa397456cb475d86aa842f848c9f3e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6bd3fd3959e84eddaf03acb4cade42c4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c7b10b88f2214064925d980b0466fb9d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5c12481a504e43fd8a2ff24393a7c617": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "852d273e2e5740fbb102e2a1c8dc1459": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d99c983ccb9f4c1f80d66d2400f660bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}