diff --git a/tutorials/27_First_RAG_Pipeline.ipynb b/tutorials/27_First_RAG_Pipeline.ipynb index d417b35..a730e47 100644 --- a/tutorials/27_First_RAG_Pipeline.ipynb +++ b/tutorials/27_First_RAG_Pipeline.ipynb @@ -70,30 +70,6 @@ "pip install \"sentence-transformers>=3.0.0\"" ] }, - { - "cell_type": "markdown", - "metadata": { - "id": "Wl_jYERtO-qa" - }, - "source": [ - "### Enabling Telemetry\n", - "\n", - "Knowing you're using this tutorial helps us decide where to invest our efforts to build a better product but you can always opt out by commenting the following line. See [Telemetry](https://docs.haystack.deepset.ai/docs/enabling-telemetry) for more details." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "id": "A76B4S49O-qa" - }, - "outputs": [], - "source": [ - "from haystack.telemetry import tutorial_running\n", - "\n", - "tutorial_running(27)" - ] - }, { "cell_type": "markdown", "metadata": { diff --git a/tutorials/28_Structured_Output_With_Loop.ipynb b/tutorials/28_Structured_Output_With_Loop.ipynb index a41271d..5041dbc 100644 --- a/tutorials/28_Structured_Output_With_Loop.ipynb +++ b/tutorials/28_Structured_Output_With_Loop.ipynb @@ -73,39 +73,6 @@ "pip install colorama" ] }, - { - "cell_type": "markdown", - "metadata": { - "id": "nTA5fdvCLMKD" - }, - "source": [ - "### Enabling Telemetry\n", - "\n", - "Enable telemetry to let us know you're using this tutorial. (You can always opt out by commenting out this line). For details, see [Telemetry](https://docs.haystack.deepset.ai/docs/telemetry)." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "id": "Apay3QSQLKdM" - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/amna.mubashar/Library/Python/3.9/lib/python/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], - "source": [ - "from haystack.telemetry import tutorial_running\n", - "\n", - "tutorial_running(28)" - ] - }, { "cell_type": "markdown", "metadata": { diff --git a/tutorials/29_Serializing_Pipelines.ipynb b/tutorials/29_Serializing_Pipelines.ipynb index 7347cca..898092d 100644 --- a/tutorials/29_Serializing_Pipelines.ipynb +++ b/tutorials/29_Serializing_Pipelines.ipynb @@ -70,30 +70,6 @@ "pip install haystack-ai" ] }, - { - "cell_type": "markdown", - "metadata": { - "id": "MhnSGxXWHNsD" - }, - "source": [ - "### Enabling Telemetry\n", - "\n", - "Knowing you're using this tutorial helps us decide where to invest our efforts to build a better product but you can always opt out by commenting the following line. See [Telemetry](https://docs.haystack.deepset.ai/docs/enabling-telemetry) for more details." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "ikIM1o9cHNcS" - }, - "outputs": [], - "source": [ - "from haystack.telemetry import tutorial_running\n", - "\n", - "tutorial_running(29)" - ] - }, { "cell_type": "markdown", "metadata": { diff --git a/tutorials/30_File_Type_Preprocessing_Index_Pipeline.ipynb b/tutorials/30_File_Type_Preprocessing_Index_Pipeline.ipynb index 537015d..c3b5740 100644 --- a/tutorials/30_File_Type_Preprocessing_Index_Pipeline.ipynb +++ b/tutorials/30_File_Type_Preprocessing_Index_Pipeline.ipynb @@ -76,39 +76,6 @@ "pip install gdown" ] }, - { - "cell_type": "markdown", - "metadata": { - "id": "HnXumz7EarJx" - }, - "source": [ - "### Enabling Telemetry\n", - "\n", - "Knowing you’re using this tutorial helps us decide where to invest our efforts to build a better product but you can always opt out by commenting the following line. See [Telemetry](https://docs.haystack.deepset.ai/docs/enabling-telemetry) for more details." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "id": "CkvJIU7FmDf9" - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/amna.mubashar/Library/Python/3.9/lib/python/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], - "source": [ - "from haystack.telemetry import tutorial_running\n", - "\n", - "tutorial_running(30)" - ] - }, { "cell_type": "markdown", "metadata": { diff --git a/tutorials/31_Metadata_Filtering.ipynb b/tutorials/31_Metadata_Filtering.ipynb index 7de9795..eca6398 100644 --- a/tutorials/31_Metadata_Filtering.ipynb +++ b/tutorials/31_Metadata_Filtering.ipynb @@ -68,30 +68,6 @@ "pip install haystack-ai" ] }, - { - "cell_type": "markdown", - "metadata": { - "id": "07gGBcxBgi8Y" - }, - "source": [ - "### Enabling Telemetry\n", - "\n", - "Knowing you're using this tutorial helps us decide where to invest our efforts to build a better product but you can always opt out by commenting the following line. See [Telemetry](https://docs.haystack.deepset.ai/docs/enabling-telemetry) for more details." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "PtJZlNtkHZr-" - }, - "outputs": [], - "source": [ - "from haystack.telemetry import tutorial_running\n", - "\n", - "tutorial_running(31)" - ] - }, { "cell_type": "markdown", "metadata": { diff --git a/tutorials/32_Classifying_Documents_and_Queries_by_Language.ipynb b/tutorials/32_Classifying_Documents_and_Queries_by_Language.ipynb index 083cb54..6ab4eb6 100644 --- a/tutorials/32_Classifying_Documents_and_Queries_by_Language.ipynb +++ b/tutorials/32_Classifying_Documents_and_Queries_by_Language.ipynb @@ -71,30 +71,6 @@ "pip install langdetect" ] }, - { - "cell_type": "markdown", - "metadata": { - "id": "32oB-HJlGXmY" - }, - "source": [ - "### Enabling Telemetry\n", - "\n", - "Knowing you're using this tutorial helps us decide where to invest our efforts to build a better product but you can always opt out by commenting the following line. See [Telemetry](https://docs.haystack.deepset.ai/docs/enabling-telemetry) for more details." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "Ubr7yVt6Gbnj" - }, - "outputs": [], - "source": [ - "from haystack.telemetry import tutorial_running\n", - "\n", - "tutorial_running(32)" - ] - }, { "cell_type": "markdown", "metadata": { diff --git a/tutorials/33_Hybrid_Retrieval.ipynb b/tutorials/33_Hybrid_Retrieval.ipynb index 764fb14..1e7cbf8 100644 --- a/tutorials/33_Hybrid_Retrieval.ipynb +++ b/tutorials/33_Hybrid_Retrieval.ipynb @@ -67,30 +67,6 @@ "pip install accelerate" ] }, - { - "cell_type": "markdown", - "metadata": { - "id": "CJBcPNbBlXQq" - }, - "source": [ - "### Enabling Telemetry\n", - "\n", - "Knowing you're using this tutorial helps us decide where to invest our efforts to build a better product but you can always opt out by commenting the following line. See [Telemetry](https://docs.haystack.deepset.ai/docs/telemetry) for more details." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "lUbTGVo4lXQv" - }, - "outputs": [], - "source": [ - "from haystack.telemetry import tutorial_running\n", - "\n", - "tutorial_running(33)" - ] - }, { "cell_type": "markdown", "metadata": { diff --git a/tutorials/34_Extractive_QA_Pipeline.ipynb b/tutorials/34_Extractive_QA_Pipeline.ipynb index 756fcff..255ca79 100644 --- a/tutorials/34_Extractive_QA_Pipeline.ipynb +++ b/tutorials/34_Extractive_QA_Pipeline.ipynb @@ -63,28 +63,6 @@ "pip install haystack-ai accelerate \"sentence-transformers>=3.0.0\" \"datasets>=2.6.1\"" ] }, - { - "cell_type": "markdown", - "metadata": { - "id": "bo6LztWXURaA" - }, - "source": [ - "Knowing you’re using this tutorial helps us decide where to invest our efforts to build a better product but you can always opt out by commenting the following line. See [Telemetry](https://docs.haystack.deepset.ai/docs/enabling-telemetry) for more details." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "y_btxLmYUUM9" - }, - "outputs": [], - "source": [ - "from haystack.telemetry import tutorial_running\n", - "\n", - "tutorial_running(34)" - ] - }, { "cell_type": "markdown", "metadata": { diff --git a/tutorials/35_Evaluating_RAG_Pipelines.ipynb b/tutorials/35_Evaluating_RAG_Pipelines.ipynb index 6bdee9a..0fb9efd 100644 --- a/tutorials/35_Evaluating_RAG_Pipelines.ipynb +++ b/tutorials/35_Evaluating_RAG_Pipelines.ipynb @@ -95,30 +95,6 @@ "pip install \"sentence-transformers>=3.0.0\"" ] }, - { - "cell_type": "markdown", - "metadata": { - "id": "Wl_jYERtO-qa" - }, - "source": [ - "### Enabling Telemetry\n", - "\n", - "Knowing you're using this tutorial helps us decide where to invest our efforts to build a better product but you can always opt out by commenting the following line. See [Telemetry](https://docs.haystack.deepset.ai/docs/enabling-telemetry) for more details." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "A76B4S49O-qa" - }, - "outputs": [], - "source": [ - "from haystack.telemetry import tutorial_running\n", - "\n", - "tutorial_running(35)" - ] - }, { "cell_type": "markdown", "metadata": { @@ -1410,7 +1386,7 @@ } ], "source": [ - "results_df = evaluation_result.detailed_report(output_format='df')\n", + "results_df = evaluation_result.detailed_report(output_format=\"df\")\n", "results_df" ] }, diff --git a/tutorials/36_Building_Fallbacks_with_Conditional_Routing.ipynb b/tutorials/36_Building_Fallbacks_with_Conditional_Routing.ipynb index 1d66004..7a591a4 100644 --- a/tutorials/36_Building_Fallbacks_with_Conditional_Routing.ipynb +++ b/tutorials/36_Building_Fallbacks_with_Conditional_Routing.ipynb @@ -73,30 +73,6 @@ "pip install haystack-ai" ] }, - { - "cell_type": "markdown", - "metadata": { - "id": "WBkJ7d3hZkOJ" - }, - "source": [ - "### Enable Telemetry\n", - "\n", - "Knowing you're using this tutorial helps us decide where to invest our efforts to build a better product but you can always opt out by commenting the following line. See [Telemetry](https://docs.haystack.deepset.ai/docs/telemetry) for more details." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "HvrOixzzZmMi" - }, - "outputs": [], - "source": [ - "from haystack.telemetry import tutorial_running\n", - "\n", - "tutorial_running(36)" - ] - }, { "cell_type": "markdown", "metadata": { diff --git a/tutorials/37_Simplifying_Pipeline_Inputs_with_Multiplexer.ipynb b/tutorials/37_Simplifying_Pipeline_Inputs_with_Multiplexer.ipynb index 27478b7..59b5add 100644 --- a/tutorials/37_Simplifying_Pipeline_Inputs_with_Multiplexer.ipynb +++ b/tutorials/37_Simplifying_Pipeline_Inputs_with_Multiplexer.ipynb @@ -70,30 +70,6 @@ "pip install haystack-ai \"huggingface_hub>=0.23.0, <0.28.0\"" ] }, - { - "cell_type": "markdown", - "metadata": { - "id": "3N_97P0OV9cx" - }, - "source": [ - "### Enable Telemetry\n", - "\n", - "Knowing you're using this tutorial helps us decide where to invest our efforts to build a better product but you can always opt out by commenting the following line. See [Telemetry](https://docs.haystack.deepset.ai/docs/telemetry) for more details." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "BKilNUd8V_Uc" - }, - "outputs": [], - "source": [ - "from haystack.telemetry import tutorial_running\n", - "\n", - "tutorial_running(37)" - ] - }, { "cell_type": "markdown", "metadata": { diff --git a/tutorials/39_Embedding_Metadata_for_Improved_Retrieval.ipynb b/tutorials/39_Embedding_Metadata_for_Improved_Retrieval.ipynb index f70dc25..8497cba 100644 --- a/tutorials/39_Embedding_Metadata_for_Improved_Retrieval.ipynb +++ b/tutorials/39_Embedding_Metadata_for_Improved_Retrieval.ipynb @@ -57,26 +57,6 @@ "pip install haystack-ai wikipedia sentence-transformers" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Enable Telemetry\n", - "\n", - "Knowing you're using this tutorial helps us decide where to invest our efforts to build a better product but you can always opt out by commenting the following line. See [Telemetry](https://docs.haystack.deepset.ai/docs/telemetry) for more details." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from haystack.telemetry import tutorial_running\n", - "\n", - "tutorial_running(39)" - ] - }, { "cell_type": "markdown", "metadata": {}, diff --git a/tutorials/40_Building_Chat_Application_with_Function_Calling.ipynb b/tutorials/40_Building_Chat_Application_with_Function_Calling.ipynb index 7f2b722..340aff9 100644 --- a/tutorials/40_Building_Chat_Application_with_Function_Calling.ipynb +++ b/tutorials/40_Building_Chat_Application_with_Function_Calling.ipynb @@ -61,30 +61,6 @@ "pip install \"sentence-transformers>=3.0.0\"" ] }, - { - "cell_type": "markdown", - "metadata": { - "id": "XwlBLN-GwuuU" - }, - "source": [ - "### Enable Telemetry\n", - "\n", - "Knowing you're using this tutorial helps us decide where to invest our efforts to build a better product but you can always opt out by commenting the following line. See [Telemetry](https://docs.haystack.deepset.ai/docs/telemetry) for more details." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "id": "9V0-VuEgwt2u" - }, - "outputs": [], - "source": [ - "from haystack.telemetry import tutorial_running\n", - "\n", - "tutorial_running(40)" - ] - }, { "cell_type": "markdown", "metadata": { diff --git a/tutorials/41_Query_Classification_with_TransformersTextRouter_and_TransformersZeroShotTextRouter.ipynb b/tutorials/41_Query_Classification_with_TransformersTextRouter_and_TransformersZeroShotTextRouter.ipynb index db03918..6785599 100644 --- a/tutorials/41_Query_Classification_with_TransformersTextRouter_and_TransformersZeroShotTextRouter.ipynb +++ b/tutorials/41_Query_Classification_with_TransformersTextRouter_and_TransformersZeroShotTextRouter.ipynb @@ -1,7518 +1,7499 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "O-W2ZQ6CN-gZ" - }, - "source": [ - "# Tutorial: Query Classification with TransformersTextRouter and TransformersZeroShotTextRouter\n", - "\n", - "- **Level**: Intermediate\n", - "- **Time to complete**: 15 minutes\n", - "- **Components Used**: [`TransformersZeroShotTextRouter`](https://docs.haystack.deepset.ai/docs/transformerszeroshottextrouter), [`TransformersTextRouter`](https://docs.haystack.deepset.ai/docs/transformerstextrouter), [`InMemoryDocumentStore`](https://docs.haystack.deepset.ai/docs/inmemorydocumentstore), [`SentenceTransformersDocumentEmbedder`](https://docs.haystack.deepset.ai/docs/sentencetransformersdocumentembedder), [`SentenceTransformersTextEmbedder`](https://docs.haystack.deepset.ai/docs/sentencetransformerstextembedder), [`InMemoryEmbeddingRetriever`](https://docs.haystack.deepset.ai/docs/inmemoryembeddingretriever), [`InMemoryBM25Retriever`](https://docs.haystack.deepset.ai/docs/inmemorybm25retriever)\n", - "- **Goal**: After completing this tutorial, you will have learned about TransformersZeroShotTextRouter and TransformersTextRouter and how to use them in a pipeline.\n", - "\n", - "## Overview\n", - "\n", - "One of the great benefits of using state-of-the-art NLP models like those available in Haystack is that it allows users to state their queries as *plain natural language questions*: rather than trying to come up with just the right set of keywords to find the answer to their question, users can simply ask their question in much the same way that they would ask it of a (very knowledgeable!) person.\n", - "\n", - "But just because users *can* ask their questions in \"plain English\" (or \"plain German\", etc.), that doesn't mean they always *will*. For instance, users might input a few keywords rather than a complete question because they don't understand the pipeline's full capabilities or are so accustomed to keyword search. While a standard Haystack pipeline might handle such queries with reasonable accuracy, for a variety of reasons we still might prefer that our pipeline is sensitive to the type of query it is receiving, so that it behaves differently when a user inputs, say, a collection of keywords instead of a question. For this reason, Haystack comes with built-in capabilities to distinguish between types of text inputs, such as **keyword queries**, **interrogative queries(questions)**, and **statement queries**.\n", - "\n", - "Given a text input, classification models ouput a label, which can be LABEL_0,\n", - "LABEL_1, ... depending on how the model was trained. Haystack's TransformersTextRouter component uses classification models and then routes the text to an output branch that is named after the label.\n", - "\n", - "In this tutorial you will learn how to use TransformersTextRouter and TransformersZeroShotTextRouter to branch your Haystack pipeline based on the type of query it receives:\n", - "\n", - "1. **Keyword vs. Question/Statement** — routes a query into one of two branches depending on whether it is a full question/statement or a collection of keywords.\n", - "\n", - "2. **Question vs. Statement** — routes a natural language query into one of two branches depending on whether it is a question or a statement.\n", - "\n", - "With TransformersTextRouter, it's also possible to route queries based on your custom classification models. With TransformersZeroShotTextRouter you can even do zero-shot classification, which means you can specify custom classes but don't need any custom model.\n", - "\n", - "With all of that explanation out of the way, let's dive in!" - ] + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "O-W2ZQ6CN-gZ" + }, + "source": [ + "# Tutorial: Query Classification with TransformersTextRouter and TransformersZeroShotTextRouter\n", + "\n", + "- **Level**: Intermediate\n", + "- **Time to complete**: 15 minutes\n", + "- **Components Used**: [`TransformersZeroShotTextRouter`](https://docs.haystack.deepset.ai/docs/transformerszeroshottextrouter), [`TransformersTextRouter`](https://docs.haystack.deepset.ai/docs/transformerstextrouter), [`InMemoryDocumentStore`](https://docs.haystack.deepset.ai/docs/inmemorydocumentstore), [`SentenceTransformersDocumentEmbedder`](https://docs.haystack.deepset.ai/docs/sentencetransformersdocumentembedder), [`SentenceTransformersTextEmbedder`](https://docs.haystack.deepset.ai/docs/sentencetransformerstextembedder), [`InMemoryEmbeddingRetriever`](https://docs.haystack.deepset.ai/docs/inmemoryembeddingretriever), [`InMemoryBM25Retriever`](https://docs.haystack.deepset.ai/docs/inmemorybm25retriever)\n", + "- **Goal**: After completing this tutorial, you will have learned about TransformersZeroShotTextRouter and TransformersTextRouter and how to use them in a pipeline.\n", + "\n", + "## Overview\n", + "\n", + "One of the great benefits of using state-of-the-art NLP models like those available in Haystack is that it allows users to state their queries as *plain natural language questions*: rather than trying to come up with just the right set of keywords to find the answer to their question, users can simply ask their question in much the same way that they would ask it of a (very knowledgeable!) person.\n", + "\n", + "But just because users *can* ask their questions in \"plain English\" (or \"plain German\", etc.), that doesn't mean they always *will*. For instance, users might input a few keywords rather than a complete question because they don't understand the pipeline's full capabilities or are so accustomed to keyword search. While a standard Haystack pipeline might handle such queries with reasonable accuracy, for a variety of reasons we still might prefer that our pipeline is sensitive to the type of query it is receiving, so that it behaves differently when a user inputs, say, a collection of keywords instead of a question. For this reason, Haystack comes with built-in capabilities to distinguish between types of text inputs, such as **keyword queries**, **interrogative queries(questions)**, and **statement queries**.\n", + "\n", + "Given a text input, classification models ouput a label, which can be LABEL_0,\n", + "LABEL_1, ... depending on how the model was trained. Haystack's TransformersTextRouter component uses classification models and then routes the text to an output branch that is named after the label.\n", + "\n", + "In this tutorial you will learn how to use TransformersTextRouter and TransformersZeroShotTextRouter to branch your Haystack pipeline based on the type of query it receives:\n", + "\n", + "1. **Keyword vs. Question/Statement** — routes a query into one of two branches depending on whether it is a full question/statement or a collection of keywords.\n", + "\n", + "2. **Question vs. Statement** — routes a natural language query into one of two branches depending on whether it is a question or a statement.\n", + "\n", + "With TransformersTextRouter, it's also possible to route queries based on your custom classification models. With TransformersZeroShotTextRouter you can even do zero-shot classification, which means you can specify custom classes but don't need any custom model.\n", + "\n", + "With all of that explanation out of the way, let's dive in!" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "yaaKv3_ZN-gb" + }, + "source": [ + "\n", + "## Preparing the Colab Environment\n", + "\n", + "- [Enable GPU Runtime](https://docs.haystack.deepset.ai/docs/enabling-gpu-acceleration#enabling-the-gpu-in-colab)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "TNlqD5HeN-gc" + }, + "source": [ + "## Installing Haystack\n", + "\n", + "To start, install the latest release of Haystack with `pip`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true, + "id": "CjA5n5lMN-gd" + }, + "outputs": [], + "source": [ + "%%bash\n", + "\n", + "pip install --upgrade pip\n", + "pip install haystack-ai torch sentencepiece datasets sentence-transformers" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "sJcWRK4Hwyx2" + }, + "source": [ + "## Trying out TransformersTextRouter\n", + "\n", + "Before integrating a TransformersTextRouter into the pipeline, test it out on its own and see what it actually does. First, initiate a simple, out-of-the-box **keyword vs. question/statement** TransformersTextRouter, which uses the [shahrukhx01/bert-mini-finetune-question-detection](https://huggingface.co/shahrukhx01/bert-mini-finetune-question-detection) model.\n", + "For this model, LABEL_0 corresponds to keyword queries and LABEL_1 to non-keyword queries." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true, + "id": "XhPMEqBzxA8V" + }, + "outputs": [], + "source": [ + "from haystack.components.routers import TransformersTextRouter\n", + "\n", + "text_router = TransformersTextRouter(model=\"shahrukhx01/bert-mini-finetune-question-detection\")\n", + "text_router.warm_up()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1NHjy9aa9FKx" + }, + "source": [ + "Now feed some queries into this text router. Test with one keyword query, one interrogative query, and one statement query. Note that you don't need to use any punctuation, such as question marks, for the text router to make the right decision." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "id": "Ks7qdULR8J13" + }, + "outputs": [], + "source": [ + "queries = [\n", + " \"Arya Stark father\", # Keyword Query\n", + " \"Who was the father of Arya Stark\", # Interrogative Query\n", + " \"Lord Eddard was the father of Arya Stark\", # Statement Query\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "UbKlyXcNj-nx" + }, + "source": [ + "Below, you can see what the text router does with these queries: it correctly determines that \"Arya Stark father\" is a keyword query and sends it to the branch LABEL_0. It also correctly classifies both the interrogative query \"Who was the father of Arya Stark\" and the statement query \"Lord Eddard was the father of Arya Stark\" as non-keyword queries, and sends them to branch 1." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 91 }, + "id": "o2Y4jvQ7U4fK", + "outputId": "b03ccf83-d451-4341-aa4b-9d2c9fc6ac3c" + }, + "outputs": [ { - "cell_type": "markdown", - "metadata": { - "id": "yaaKv3_ZN-gb" - }, - "source": [ - "\n", - "## Preparing the Colab Environment\n", - "\n", - "- [Enable GPU Runtime](https://docs.haystack.deepset.ai/docs/enabling-gpu-acceleration#enabling-the-gpu-in-colab)\n" - ] + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.10/dist-packages/transformers/pipelines/text_classification.py:104: UserWarning: `return_all_scores` is now deprecated, if want a similar functionality use `top_k=None` instead of `return_all_scores=True` or `top_k=1` instead of `return_all_scores=False`.\n", + " warnings.warn(\n" + ] }, { - "cell_type": "markdown", - "metadata": { - "id": "TNlqD5HeN-gc" + "data": { + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "string" }, - "source": [ - "## Installing Haystack\n", - "\n", - "To start, install the latest release of Haystack with `pip`:" + "text/plain": [ + "'LABEL_0'" ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "result = text_router.run(text=queries[0])\n", + "next(iter(result))" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 143 }, + "id": "NYROmSHnE4zp", + "outputId": "e6e15b35-452f-4f43-a1c7-af2573962289" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true, - "id": "CjA5n5lMN-gd" + "data": { + "application/vnd.google.colaboratory.intrinsic+json": { + "summary": "{\n \"name\": \"pd\",\n \"rows\": 3,\n \"fields\": [\n {\n \"column\": \"Query\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 3,\n \"samples\": [\n \"Arya Stark father\",\n \"Who was the father of Arya Stark\",\n \"Lord Eddard was the father of Arya Stark\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Output Branch\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 2,\n \"samples\": [\n \"LABEL_1\",\n \"LABEL_0\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Class\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 2,\n \"samples\": [\n \"Question/Statement\",\n \"Keyword Query\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}", + "type": "dataframe" }, - "outputs": [], - "source": [ - "%%bash\n", - "\n", - "pip install --upgrade pip\n", - "pip install haystack-ai torch sentencepiece datasets sentence-transformers" + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
QueryOutput BranchClass
0Arya Stark fatherLABEL_0Keyword Query
1Who was the father of Arya StarkLABEL_1Question/Statement
2Lord Eddard was the father of Arya StarkLABEL_1Question/Statement
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "\n", + "
\n", + "
\n" + ], + "text/plain": [ + " Query Output Branch Class\n", + "0 Arya Stark father LABEL_0 Keyword Query\n", + "1 Who was the father of Arya Stark LABEL_1 Question/Statement\n", + "2 Lord Eddard was the father of Arya Stark LABEL_1 Question/Statement" ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "results = {\"Query\": [], \"Output Branch\": [], \"Class\": []}\n", + "\n", + "for query in queries:\n", + " result = text_router.run(text=query)\n", + " results[\"Query\"].append(query)\n", + " results[\"Output Branch\"].append(next(iter(result)))\n", + " results[\"Class\"].append(\"Keyword Query\" if next(iter(result)) == \"LABEL_0\" else \"Question/Statement\")\n", + "\n", + "pd.DataFrame.from_dict(results)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qyotwqKKbIpb" + }, + "source": [ + "Next, you will illustrate a **question vs. statement** TransformersTextRouter using [shahrukhx01/question-vs-statement-classifier](https://huggingface.co/shahrukhx01/question-vs-statement-classifier). For this task, you need to initialize a new text router with this classification model." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 364, + "referenced_widgets": [ + "5e15e24548b044bfb864eb8cc5b712c9", + "d8c1fe2c43ec4b8b95092488f138cd39", + "4d3cfdeb1bd24db8824f00468b1635c5", + "f9052e0559544554b01b4afb1206dce8", + "6d73cba2cf3f4fb9b25934700736d41b", + "7bcc9b32caf84596991cb41bb5931fad", + "2b0ea9f3052043ff9f239fcf108c2f6d", + "4603de989e4640efbf7c379e5c7640dc", + "1f1be98d3b294254a0ac88aafd6ce67c", + "d58f49a028f6411783af61fecbc68a3c", + "26e71a1ef9b740e0b6c59a088c17b241", + "4e664743e8e24c40aab9a4b4bf4d4ec6", + "1c1acd6c555e48a088036c3c4b19d6a8", + "52a5a268f56a41c3bd39db3a05268192", + "a58e79dda9ae4548b7b774282dac7559", + "b2c23aa910c146869303f99f29d6ac14", + "4de8ed8dfc8a47f694ddcf7fc05470ce", + "6135c2c5b28c4189b4c5600ae11c7f93", + "215cf162756c4da19ec6f84792a39b69", + "c2ad3379af7b4a83932c670117b6ac4d", + "6b7f104b9356477e81eba214768fc9d9", + "dec1f43ffab34f3c96ce08670dc5e848", + "172c9b77254048e492f902e4701ae4f4", + "138d4dda99b4450a88a311faea032010", + "4a26297c8a634c68b1024bc178cfa1d3", + "f8ab59fad4d54c11b8ecb28a135f77ee", + "1e3823ff48a64f719da05bb5522477e3", + "fd822ca15aab4929a8a61adfce6e6479", + "4f02f9ae1623496f8754406eb24527b1", + "c498bcf3c5aa4251a3969d26b1b0e770", + "99858d8e2b67469db0bcb1f87a029c99", + "36fe5bad3e4e434c8128132da61af293", + "444170e3f8834d21983f89095aebdae3", + "c6b33c3c33074511a8184c57c37e2128", + "1ca743af07ab42829fd961acdd641a4a", + "ae6be99a77af40a5aa47314f22804e01", + "c8d59f70340f4e9d9e63932bb7871f32", + "3a5f55591c4d4e6d9f2439898a0780c2", + "90dfee714fa14024aa0235709c8da661", + "1f711847a2314067a2ffd5fd60dbd959", + "b0453017029f481abdaec1fd5c7308fa", + "011a4dc918c54bc19e48e7f0c99c95f2", + "a068875795d44c5daa2eff8e74bd021d", + "425ebd00eaa54edfb531f00e08cc90cf", + "0c913d57bbe54ac59702e61c4ba753a9", + "20177ae3a9f14622a1fef3196159cb44", + "b24e12ce7fba4841b76ed518d4659356", + "770d46aa7dca4ad497d3d5c64ce5147e", + "f9d574e591e64ed8b764103d11c437c0", + "a203aaa4067a4335b9f61bb81b6006e5", + "17d4d36595184416bdbbb2722d324127", + "1576424bae8d4428805f27f725184d09", + "821ebc0e98a3460fa5844658a4f66405", + "13db9db575bf47db8f23becbd9e01019", + "86d6f5dbdaa24e648a0e1600b7f2d115" + ] }, + "id": "4CsZmiDZbFhs", + "outputId": "7669564e-4b23-4dcf-f38e-873b03cebf63" + }, + "outputs": [ { - "cell_type": "markdown", - "metadata": { - "id": "2NaWOkfTvLgn" + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "5e15e24548b044bfb864eb8cc5b712c9", + "version_major": 2, + "version_minor": 0 }, - "source": [ - "### Enabling Telemetry\n", - "Knowing you're using this tutorial helps us decide where to invest our efforts to build a better product but you can always opt out by commenting the following line. See [Telemetry](https://docs.haystack.deepset.ai/docs/telemetry) for more details." + "text/plain": [ + "config.json: 0%| | 0.00/619 [00:00\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
QueryOutput BranchClass
0Who was the father of Arya StarkLABEL_1Question
1Lord Eddard was the father of Arya StarkLABEL_0Statement
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "\n", + "
\n", + " \n" + ], + "text/plain": [ + " Query Output Branch Class\n", + "0 Who was the father of Arya Stark LABEL_1 Question\n", + "1 Lord Eddard was the father of Arya Stark LABEL_0 Statement" ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "text_router = TransformersTextRouter(model=\"shahrukhx01/question-vs-statement-classifier\")\n", + "text_router.warm_up()\n", + "\n", + "queries = [\n", + " \"Who was the father of Arya Stark\", # Interrogative Query\n", + " \"Lord Eddard was the father of Arya Stark\", # Statement Query\n", + "]\n", + "\n", + "results = {\"Query\": [], \"Output Branch\": [], \"Class\": []}\n", + "\n", + "for query in queries:\n", + " result = text_router.run(text=query)\n", + " results[\"Query\"].append(query)\n", + " results[\"Output Branch\"].append(next(iter(result)))\n", + " results[\"Class\"].append(\"Question\" if next(iter(result)) == \"LABEL_1\" else \"Statement\")\n", + "\n", + "pd.DataFrame.from_dict(results)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "U_FeMveXVvAh" + }, + "source": [ + "Again, the text router send the question and the statement to the expected output branches." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "fzxOEXENvLgt" + }, + "source": [ + "## Custom Use Cases for Text Classification\n", + "\n", + "TransformersTextRouter is very flexible and also supports other options for classifying texts beyond distinguishing keyword queries from interrogative queries.\n", + "For example, you may be interested in detecting the sentiment of a text or classifying the topics. You can do this by loading a custom classification model from the Hugging Face Hub or by using zero-shot classification with TransformersZeroShotTextRouter.\n", + "\n", + "- Traditional text classification models are trained to predict one of a few \"hard-coded\" classes and require a dedicated training dataset. In the Hugging Face Hub, you can find many pre-trained models, maybe even related to your domain of interest.\n", + "- Zero-shot classification is very versatile: by choosing a suitable base transformer, you can classify the text without any training dataset. You just have to provide the candidate categories." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "SHCSX05NvLgt" + }, + "source": [ + "### Custom Classification Models with TransformersTextRouter\n", + "For this use case, you can use a public model available in the Hugging Face Hub. For example, if you want to classify the sentiment of the queries, you can choose an appropriate model, such as [`cardiffnlp/twitter-roberta-base-sentiment`](https://huggingface.co/cardiffnlp/twitter-roberta-base-sentiment)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "qUOfTzi5vLgt" + }, + "outputs": [], + "source": [ + "text_router = TransformersTextRouter(model=\"cardiffnlp/twitter-roberta-base-sentiment\")\n", + "text_router.warm_up()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "id": "8hQUejENvLgt" + }, + "outputs": [], + "source": [ + "queries = [\n", + " \"What's the answer?\", # neutral query\n", + " \"Would you be so lovely to tell me the answer?\", # positive query\n", + " \"Can you give me the damn right answer for once??\", # negative query\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 199 }, + "id": "fX6nUR8OvLgu", + "outputId": "7d9a9d51-ebe3-4d72-a949-9b5854e544c6" + }, + "outputs": [ { - "cell_type": "code", - "source": [ - "result = text_router.run(text=queries[0])\n", - "next(iter(result))" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 91 - }, - "id": "o2Y4jvQ7U4fK", - "outputId": "b03ccf83-d451-4341-aa4b-9d2c9fc6ac3c" - }, - "execution_count": 5, - "outputs": [ - { - "output_type": "stream", - "name": "stderr", - "text": [ - "/usr/local/lib/python3.10/dist-packages/transformers/pipelines/text_classification.py:104: UserWarning: `return_all_scores` is now deprecated, if want a similar functionality use `top_k=None` instead of `return_all_scores=True` or `top_k=1` instead of `return_all_scores=False`.\n", - " warnings.warn(\n" - ] - }, - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "'LABEL_0'" - ], - "application/vnd.google.colaboratory.intrinsic+json": { - "type": "string" - } - }, - "metadata": {}, - "execution_count": 5 - } - ] + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.10/dist-packages/transformers/pipelines/text_classification.py:104: UserWarning: `return_all_scores` is now deprecated, if want a similar functionality use `top_k=None` instead of `return_all_scores=True` or `top_k=1` instead of `return_all_scores=False`.\n", + " warnings.warn(\n" + ] }, { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "id": "NYROmSHnE4zp", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 143 - }, - "outputId": "e6e15b35-452f-4f43-a1c7-af2573962289" + "data": { + "application/vnd.google.colaboratory.intrinsic+json": { + "summary": "{\n \"name\": \"pd\",\n \"rows\": 3,\n \"fields\": [\n {\n \"column\": \"Query\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 3,\n \"samples\": [\n \"What's the answer?\",\n \"Would you be so lovely to tell me the answer?\",\n \"Can you give me the damn right answer for once??\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Output Branch\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 3,\n \"samples\": [\n \"LABEL_1\",\n \"LABEL_2\",\n \"LABEL_0\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Class\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 3,\n \"samples\": [\n \"neutral\",\n \"positive\",\n \"negative\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}", + "type": "dataframe" }, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - " Query Output Branch Class\n", - "0 Arya Stark father LABEL_0 Keyword Query\n", - "1 Who was the father of Arya Stark LABEL_1 Question/Statement\n", - "2 Lord Eddard was the father of Arya Stark LABEL_1 Question/Statement" - ], - "text/html": [ - "\n", - "
\n", - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
QueryOutput BranchClass
0Arya Stark fatherLABEL_0Keyword Query
1Who was the father of Arya StarkLABEL_1Question/Statement
2Lord Eddard was the father of Arya StarkLABEL_1Question/Statement
\n", - "
\n", - "
\n", - "\n", - "
\n", - " \n", - "\n", - " \n", - "\n", - " \n", - "
\n", - "\n", - "\n", - "
\n", - " \n", - "\n", - "\n", - "\n", - " \n", - "
\n", - "\n", - "
\n", - "
\n" - ], - "application/vnd.google.colaboratory.intrinsic+json": { - "type": "dataframe", - "summary": "{\n \"name\": \"pd\",\n \"rows\": 3,\n \"fields\": [\n {\n \"column\": \"Query\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 3,\n \"samples\": [\n \"Arya Stark father\",\n \"Who was the father of Arya Stark\",\n \"Lord Eddard was the father of Arya Stark\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Output Branch\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 2,\n \"samples\": [\n \"LABEL_1\",\n \"LABEL_0\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Class\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 2,\n \"samples\": [\n \"Question/Statement\",\n \"Keyword Query\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}" - } - }, - "metadata": {}, - "execution_count": 6 - } + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
QueryOutput BranchClass
0What's the answer?LABEL_1neutral
1Would you be so lovely to tell me the answer?LABEL_2positive
2Can you give me the damn right answer for once??LABEL_0negative
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "\n", + "
\n", + "
\n" ], - "source": [ - "import pandas as pd\n", - "\n", - "results = {\"Query\": [], \"Output Branch\": [], \"Class\": []}\n", - "\n", - "for query in queries:\n", - " result = text_router.run(text=query)\n", - " results[\"Query\"].append(query)\n", - " results[\"Output Branch\"].append(next(iter(result)))\n", - " results[\"Class\"].append(\"Keyword Query\" if next(iter(result)) == \"LABEL_0\" else \"Question/Statement\")\n", - "\n", - "pd.DataFrame.from_dict(results)" + "text/plain": [ + " Query Output Branch Class\n", + "0 What's the answer? LABEL_1 neutral\n", + "1 Would you be so lovely to tell me the answer? LABEL_2 positive\n", + "2 Can you give me the damn right answer for once?? LABEL_0 negative" ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sent_results = {\"Query\": [], \"Output Branch\": [], \"Class\": []}\n", + "\n", + "for query in queries:\n", + " result = text_router.run(text=query)\n", + " sent_results[\"Query\"].append(query)\n", + " sent_results[\"Output Branch\"].append(next(iter(result)))\n", + " sent_results[\"Class\"].append(\n", + " {\"LABEL_0\": \"negative\", \"LABEL_1\": \"neutral\", \"LABEL_2\": \"positive\"}.get(next(iter(result)), \"Unknown\")\n", + " )\n", + "\n", + "pd.DataFrame.from_dict(sent_results)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-6R8jfJVvLgu" + }, + "source": [ + "### Zero-Shot Classification with TransformersZeroShotTextRouter\n", + "TransformersZeroShotTextRouter let's you perform zero-shot classification by providing a suitable base transformer model and **defining** the classes the model should predict.\n", + "\n", + "First, initialize a TransformersZeroShotTextRouter with some custom category labels. By default, it uses the base size zero shot classification model [MoritzLaurer/deberta-v3-base-zeroshot-v1.1-all-33](https://huggingface.co/MoritzLaurer/deberta-v3-base-zeroshot-v1.1-all-33). You can switch to the larger model [MoritzLaurer/deberta-v3-large-zeroshot-v2.0](https://huggingface.co/MoritzLaurer/deberta-v3-large-zeroshot-v2.0) and get even better results by using `TransformersZeroShotTextRouter(model=\"MoritzLaurer/deberta-v3-large-zeroshot-v2.0\")`.\n", + "\n", + "Let's look at an example. You may be interested in whether the user query is related to music or cinema. In this case, the `labels` parameter is a list containing the candidate classes and the output branches of the component are named accordingly." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "xO7sr516vLgu" + }, + "outputs": [], + "source": [ + "from haystack.components.routers import TransformersZeroShotTextRouter\n", + "\n", + "text_router = TransformersZeroShotTextRouter(labels=[\"music\", \"cinema\"])\n", + "text_router.warm_up()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "id": "lsgh0_YIvLgu" + }, + "outputs": [], + "source": [ + "queries = [\n", + " \"In which films does John Travolta appear?\", # cinema\n", + " \"What is the Rolling Stones first album?\", # music\n", + " \"Who was Sergio Leone?\", # cinema\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 161 }, + "id": "GrGR1xzmvLgu", + "outputId": "d076e942-df75-4cd4-c7c7-2ac653df882c" + }, + "outputs": [ { - "cell_type": "markdown", - "source": [ - "Next, you will illustrate a **question vs. statement** TransformersTextRouter using [shahrukhx01/question-vs-statement-classifier](https://huggingface.co/shahrukhx01/question-vs-statement-classifier). For this task, you need to initialize a new text router with this classification model." - ], - "metadata": { - "id": "qyotwqKKbIpb" - } + "name": "stderr", + "output_type": "stream", + "text": [ + "Asking to truncate to max_length but no maximum length is provided and the model has no predefined maximum length. Default to no truncation.\n" + ] }, { - "cell_type": "code", - "source": [ - "text_router = TransformersTextRouter(model=\"shahrukhx01/question-vs-statement-classifier\")\n", - "text_router.warm_up()\n", - "\n", - "queries = [\n", - " \"Who was the father of Arya Stark\", # Interrogative Query\n", - " \"Lord Eddard was the father of Arya Stark\", # Statement Query\n", - "]\n", - "\n", - "results = {\"Query\": [], \"Output Branch\": [], \"Class\": []}\n", - "\n", - "for query in queries:\n", - " result = text_router.run(text=query)\n", - " results[\"Query\"].append(query)\n", - " results[\"Output Branch\"].append(next(iter(result)))\n", - " results[\"Class\"].append(\"Question\" if next(iter(result)) == \"LABEL_1\" else \"Statement\")\n", - "\n", - "pd.DataFrame.from_dict(results)" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 364, - "referenced_widgets": [ - "5e15e24548b044bfb864eb8cc5b712c9", - "d8c1fe2c43ec4b8b95092488f138cd39", - "4d3cfdeb1bd24db8824f00468b1635c5", - "f9052e0559544554b01b4afb1206dce8", - "6d73cba2cf3f4fb9b25934700736d41b", - "7bcc9b32caf84596991cb41bb5931fad", - "2b0ea9f3052043ff9f239fcf108c2f6d", - "4603de989e4640efbf7c379e5c7640dc", - "1f1be98d3b294254a0ac88aafd6ce67c", - "d58f49a028f6411783af61fecbc68a3c", - "26e71a1ef9b740e0b6c59a088c17b241", - "4e664743e8e24c40aab9a4b4bf4d4ec6", - "1c1acd6c555e48a088036c3c4b19d6a8", - "52a5a268f56a41c3bd39db3a05268192", - "a58e79dda9ae4548b7b774282dac7559", - "b2c23aa910c146869303f99f29d6ac14", - "4de8ed8dfc8a47f694ddcf7fc05470ce", - "6135c2c5b28c4189b4c5600ae11c7f93", - "215cf162756c4da19ec6f84792a39b69", - "c2ad3379af7b4a83932c670117b6ac4d", - "6b7f104b9356477e81eba214768fc9d9", - "dec1f43ffab34f3c96ce08670dc5e848", - "172c9b77254048e492f902e4701ae4f4", - "138d4dda99b4450a88a311faea032010", - "4a26297c8a634c68b1024bc178cfa1d3", - "f8ab59fad4d54c11b8ecb28a135f77ee", - "1e3823ff48a64f719da05bb5522477e3", - "fd822ca15aab4929a8a61adfce6e6479", - "4f02f9ae1623496f8754406eb24527b1", - "c498bcf3c5aa4251a3969d26b1b0e770", - "99858d8e2b67469db0bcb1f87a029c99", - "36fe5bad3e4e434c8128132da61af293", - "444170e3f8834d21983f89095aebdae3", - "c6b33c3c33074511a8184c57c37e2128", - "1ca743af07ab42829fd961acdd641a4a", - "ae6be99a77af40a5aa47314f22804e01", - "c8d59f70340f4e9d9e63932bb7871f32", - "3a5f55591c4d4e6d9f2439898a0780c2", - "90dfee714fa14024aa0235709c8da661", - "1f711847a2314067a2ffd5fd60dbd959", - "b0453017029f481abdaec1fd5c7308fa", - "011a4dc918c54bc19e48e7f0c99c95f2", - "a068875795d44c5daa2eff8e74bd021d", - "425ebd00eaa54edfb531f00e08cc90cf", - "0c913d57bbe54ac59702e61c4ba753a9", - "20177ae3a9f14622a1fef3196159cb44", - "b24e12ce7fba4841b76ed518d4659356", - "770d46aa7dca4ad497d3d5c64ce5147e", - "f9d574e591e64ed8b764103d11c437c0", - "a203aaa4067a4335b9f61bb81b6006e5", - "17d4d36595184416bdbbb2722d324127", - "1576424bae8d4428805f27f725184d09", - "821ebc0e98a3460fa5844658a4f66405", - "13db9db575bf47db8f23becbd9e01019", - "86d6f5dbdaa24e648a0e1600b7f2d115" - ] - }, - "id": "4CsZmiDZbFhs", - "outputId": "7669564e-4b23-4dcf-f38e-873b03cebf63" + "data": { + "application/vnd.google.colaboratory.intrinsic+json": { + "summary": "{\n \"name\": \"pd\",\n \"rows\": 3,\n \"fields\": [\n {\n \"column\": \"Query\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 3,\n \"samples\": [\n \"In which films does John Travolta appear?\",\n \"What is the Rolling Stones first album?\",\n \"Who was Sergio Leone?\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Output Branch\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 2,\n \"samples\": [\n \"music\",\n \"cinema\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}", + "type": "dataframe" }, - "execution_count": 7, - "outputs": [ - { - "output_type": "display_data", - "data": { - "text/plain": [ - "config.json: 0%| | 0.00/619 [00:00\n", - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
QueryOutput BranchClass
0Who was the father of Arya StarkLABEL_1Question
1Lord Eddard was the father of Arya StarkLABEL_0Statement
\n", - "
\n", - "
\n", - "\n", - "
\n", - " \n", - "\n", - " \n", - "\n", - " \n", - "
\n", - "\n", - "\n", - "
\n", - " \n", - "\n", - "\n", - "\n", - " \n", - "
\n", - "\n", - "
\n", - " \n" - ], - "application/vnd.google.colaboratory.intrinsic+json": { - "type": "dataframe", - "summary": "{\n \"name\": \"pd\",\n \"rows\": 2,\n \"fields\": [\n {\n \"column\": \"Query\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 2,\n \"samples\": [\n \"Lord Eddard was the father of Arya Stark\",\n \"Who was the father of Arya Stark\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Output Branch\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 2,\n \"samples\": [\n \"LABEL_0\",\n \"LABEL_1\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Class\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 2,\n \"samples\": [\n \"Statement\",\n \"Question\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}" - } - }, - "metadata": {}, - "execution_count": 7 - } + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
QueryOutput Branch
0In which films does John Travolta appear?cinema
1What is the Rolling Stones first album?music
2Who was Sergio Leone?cinema
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "\n", + "
\n", + "
\n" + ], + "text/plain": [ + " Query Output Branch\n", + "0 In which films does John Travolta appear? cinema\n", + "1 What is the Rolling Stones first album? music\n", + "2 Who was Sergio Leone? cinema" ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sent_results = {\"Query\": [], \"Output Branch\": []}\n", + "\n", + "for query in queries:\n", + " result = text_router.run(text=query)\n", + " sent_results[\"Query\"].append(query)\n", + " sent_results[\"Output Branch\"].append(next(iter(result)))\n", + "\n", + "pd.DataFrame.from_dict(sent_results)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ZuGnJLMKSeNm" + }, + "source": [ + "Similar to the previous example, we can use zero-shot text classification to group questions into \"Game of Thrones\", \"Star Wars\" and \"Lord of the Rings\" related question. The number of labels is up to you!" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 161 }, + "id": "1ZULHEBVmqq2", + "outputId": "50bfdccc-d04c-40f8-d616-9da8f8356b40" + }, + "outputs": [ { - "cell_type": "markdown", - "source": [ - "Again, the text router send the question and the statement to the expected output branches." - ], - "metadata": { - "id": "U_FeMveXVvAh" - } + "name": "stderr", + "output_type": "stream", + "text": [ + "Asking to truncate to max_length but no maximum length is provided and the model has no predefined maximum length. Default to no truncation.\n" + ] }, { - "cell_type": "markdown", - "metadata": { - "id": "fzxOEXENvLgt" + "data": { + "application/vnd.google.colaboratory.intrinsic+json": { + "summary": "{\n \"name\": \"pd\",\n \"rows\": 3,\n \"fields\": [\n {\n \"column\": \"Query\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 3,\n \"samples\": [\n \"Who was the father of Arya Stark\",\n \"Who was the father of Luke Skywalker\",\n \"Who was the father of Frodo Baggins\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Output Branch\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 3,\n \"samples\": [\n \"Game of Thrones\",\n \"Star Wars\",\n \"Lord of the Rings\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}", + "type": "dataframe" }, - "source": [ - "## Custom Use Cases for Text Classification\n", - "\n", - "TransformersTextRouter is very flexible and also supports other options for classifying texts beyond distinguishing keyword queries from interrogative queries.\n", - "For example, you may be interested in detecting the sentiment of a text or classifying the topics. You can do this by loading a custom classification model from the Hugging Face Hub or by using zero-shot classification with TransformersZeroShotTextRouter.\n", - "\n", - "- Traditional text classification models are trained to predict one of a few \"hard-coded\" classes and require a dedicated training dataset. In the Hugging Face Hub, you can find many pre-trained models, maybe even related to your domain of interest.\n", - "- Zero-shot classification is very versatile: by choosing a suitable base transformer, you can classify the text without any training dataset. You just have to provide the candidate categories." + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
QueryOutput Branch
0Who was the father of Arya StarkGame of Thrones
1Who was the father of Luke SkywalkerStar Wars
2Who was the father of Frodo BagginsLord of the Rings
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "\n", + "
\n", + "
\n" + ], + "text/plain": [ + " Query Output Branch\n", + "0 Who was the father of Arya Stark Game of Thrones\n", + "1 Who was the father of Luke Skywalker Star Wars\n", + "2 Who was the father of Frodo Baggins Lord of the Rings" ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from haystack.components.routers import TransformersZeroShotTextRouter\n", + "\n", + "text_router = TransformersZeroShotTextRouter(labels=[\"Game of Thrones\", \"Star Wars\", \"Lord of the Rings\"])\n", + "text_router.warm_up()\n", + "\n", + "queries = [\n", + " \"Who was the father of Arya Stark\", # Game of Thrones\n", + " \"Who was the father of Luke Skywalker\", # Star Wars\n", + " \"Who was the father of Frodo Baggins\", # Lord of the Rings\n", + "]\n", + "\n", + "results = {\"Query\": [], \"Output Branch\": []}\n", + "\n", + "for query in queries:\n", + " result = text_router.run(text=query)\n", + " results[\"Query\"].append(query)\n", + " results[\"Output Branch\"].append(next(iter(result)))\n", + "\n", + "pd.DataFrame.from_dict(results)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Fk2kpvQR6Fa0" + }, + "source": [ + "And as you see, the question about \"Arya Stark\" is sent to branch \"Game of Thrones\", while the question about \"Luke Skywalker\" is sent to branch \"Star Wars\" and the question about \"Frodo Baggins\" is sent to \"Lord of the Rings\". This means you can have your pipeline treat questions about these universes differently." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-RgdoR9qU4G8" + }, + "source": [ + "Congratulations! 🎉 You’ve learned how TransformersZeroShotTextRouter and TransformersTextRouter work and how you can use these components individually. Now let's explore how to use them in a pipeline." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "eEwDIq9KXXke" + }, + "source": [ + "## Pipeline with Keyword vs. Question/Statement Query Classification\n", + "\n", + "Now you will create a question-answering (QA) pipeline with keyword vs. question/statement query classification and route the questions based on the classification result." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "H2VqDAVJvLgo" + }, + "source": [ + "## Fetching and Indexing Documents\n", + "\n", + "You'll start creating your question answering system by downloading the data and indexing the data with its embeddings to a DocumentStore.\n", + "\n", + "In this tutorial, you will take a simple approach to writing documents and their embeddings into the DocumentStore. For a full indexing pipeline with preprocessing, cleaning and splitting, check out our tutorial on [Preprocessing Different File Types](https://haystack.deepset.ai/tutorials/30_file_type_preprocessing_index_pipeline).\n", + "\n", + "\n", + "### Initializing the DocumentStore\n", + "\n", + "Initialize a DocumentStore to index your documents. A DocumentStore stores the Documents that the question answering system uses to find answers to your questions. In this tutorial, you'll be using the `InMemoryDocumentStore`." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "collapsed": true, + "id": "Ig7dgfdHN-gg" + }, + "outputs": [], + "source": [ + "from haystack.document_stores.in_memory import InMemoryDocumentStore\n", + "\n", + "document_store = InMemoryDocumentStore()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "dqAtAD3mVTgK" + }, + "source": [ + "> `InMemoryDocumentStore` is the simplest DocumentStore to get started with. It requires no external dependencies and it's a good option for smaller projects and debugging. But it doesn't scale up so well to larger Document collections, so it's not a good choice for production systems. To learn more about the different types of external databases that Haystack supports, see [DocumentStore Integrations](https://haystack.deepset.ai/integrations?type=Document+Store)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ZpdVE3V8VX6u" + }, + "source": [ + "The DocumentStore is now ready. Now it's time to fill it with some Documents." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "P_YLvPDCVb2z" + }, + "source": [ + "### Fetch the Data\n", + "\n", + "You'll use the Wikipedia pages of [Seven Wonders of the Ancient World](https://en.wikipedia.org/wiki/Wonders_of_the_World) as Documents. We preprocessed the data and uploaded to a Hugging Face Space: [Seven Wonders](https://huggingface.co/datasets/bilgeyucel/seven-wonders). Thus, you don't need to perform any additional cleaning or splitting.\n", + "\n", + "Fetch the data and convert it into Haystack Documents:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "esXQClYnVeoM" + }, + "outputs": [], + "source": [ + "from datasets import load_dataset\n", + "from haystack import Document\n", + "\n", + "dataset = load_dataset(\"bilgeyucel/seven-wonders\", split=\"train\")\n", + "docs = [Document(content=doc[\"content\"], meta=doc[\"meta\"]) for doc in dataset]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Ghm4qrNqVhaA" + }, + "source": [ + "### Initalize a Document Embedder\n", + "\n", + "To store your data in the DocumentStore with embeddings, initialize a [SentenceTransformersDocumentEmbedder](https://docs.haystack.deepset.ai/docs/sentencetransformersdocumentembedder) with the model name and call `warm_up()` to download the embedding model.\n", + "\n", + "> If you'd like, you can use a different [Embedder](https://docs.haystack.deepset.ai/docs/embedders) for your documents." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "hqGNjp5lVkbv" + }, + "outputs": [], + "source": [ + "from haystack.components.embedders import SentenceTransformersDocumentEmbedder\n", + "\n", + "doc_embedder = SentenceTransformersDocumentEmbedder(model=\"sentence-transformers/all-MiniLM-L6-v2\")\n", + "doc_embedder.warm_up()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "_wzljJVYWQP4" + }, + "source": [ + "### Write Documents to the DocumentStore\n", + "\n", + "Run the `doc_embedder` with the Documents. The embedder will create embeddings for each document and save these embeddings in Document object's `embedding` field. Then, you can write the Documents to the DocumentStore with `write_documents()` method." + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 67, + "referenced_widgets": [ + "fe2b5ce2eaed470892fd2de0a3bf09c8", + "1bc14d35ee3c4fd1bfe53713c410cb4e", + "db03553f0ae14d028fd1bea90f8cbc80", + "268db15fd9a3450caa5c0e5a242121c4", + "807f775eb0a249cf91367f70b0791c60", + "85c78672211245b8af111b88341192f9", + "37955fbdf27e4b278e1ed53efb00ae03", + "894091b62a084a538a39e58f93d3ea6e", + "1e2e4c55c2044fb7965649506009efc0", + "a368ee64770e47bca6980d658fad932a", + "940070f87ee745139d725399dd649392" + ] }, + "id": "yxXpVm07WS6D", + "outputId": "3862c83e-6f0a-4f2a-fd33-eaf09c274840" + }, + "outputs": [ { - "cell_type": "markdown", - "metadata": { - "id": "SHCSX05NvLgt" + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "fe2b5ce2eaed470892fd2de0a3bf09c8", + "version_major": 2, + "version_minor": 0 }, - "source": [ - "### Custom Classification Models with TransformersTextRouter\n", - "For this use case, you can use a public model available in the Hugging Face Hub. For example, if you want to classify the sentiment of the queries, you can choose an appropriate model, such as [`cardiffnlp/twitter-roberta-base-sentiment`](https://huggingface.co/cardiffnlp/twitter-roberta-base-sentiment)." + "text/plain": [ + "Batches: 0%| | 0/5 [00:00\n", + "🚅 Components\n", + " - text_router: TransformersTextRouter\n", + " - text_embedder: SentenceTransformersTextEmbedder\n", + " - embedding_retriever: InMemoryEmbeddingRetriever\n", + " - bm25_retriever: InMemoryBM25Retriever\n", + " - document_joiner: DocumentJoiner\n", + "🛤️ Connections\n", + " - text_router.LABEL_0 -> text_embedder.text (str)\n", + " - text_router.LABEL_1 -> bm25_retriever.query (str)\n", + " - text_embedder.embedding -> embedding_retriever.query_embedding (List[float])\n", + " - embedding_retriever.documents -> document_joiner.documents (List[Document])\n", + " - bm25_retriever.documents -> document_joiner.documents (List[Document])" ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from haystack import Pipeline\n", + "\n", + "query_classification_pipeline = Pipeline()\n", + "query_classification_pipeline.add_component(\"text_router\", text_router)\n", + "query_classification_pipeline.add_component(\"text_embedder\", text_embedder)\n", + "query_classification_pipeline.add_component(\"embedding_retriever\", embedding_retriever)\n", + "query_classification_pipeline.add_component(\"bm25_retriever\", bm25_retriever)\n", + "query_classification_pipeline.add_component(\"document_joiner\", document_joiner)\n", + "\n", + "query_classification_pipeline.connect(\"text_router.LABEL_0\", \"text_embedder\")\n", + "query_classification_pipeline.connect(\"text_embedder\", \"embedding_retriever\")\n", + "query_classification_pipeline.connect(\"text_router.LABEL_1\", \"bm25_retriever\")\n", + "query_classification_pipeline.connect(\"bm25_retriever\", \"document_joiner\")\n", + "query_classification_pipeline.connect(\"embedding_retriever\", \"document_joiner\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "imqRRCGTwQav" + }, + "source": [ + "### 4) Run the Pipeline\n", + "\n", + "Below, you can see how this choice affects the branching structure: the keyword query \"arya stark father\" and the question query \"Who is the father of Arya Stark?\" generate noticeably different results, a distinction that is likely due to the use of different retrievers for keyword vs. question/statement queries." + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 355, + "referenced_widgets": [ + "93e5e993baf346b39205fab6d988da84", + "0c4a42c7897d4da1b3e7b62007196854", + "24ffa535eee34c48820d1cdc107cb484", + "409eb7e9ed9e4bc1ba6fe9e324a55cdc", + "35f35dadb3c649ebba63a8cf1ea7ab99", + "dcf6276e44cd491c912d1b09502fc91b", + "63dcf941fcc94f47bb7d590a5e2f54b8", + "7c99612b86bf428d9ce4d7b4440caff4", + "1da668f10ae3483da7de191430c211fb", + "d58ed27dd0134542a43afedca9254f8d", + "f675a27210c74769a6fdafacb484e2d6" + ] }, + "collapsed": true, + "id": "fP6Cpcb-o0HK", + "outputId": "a6bfc181-c8f3-4d01-fe90-a4f5d787eae2" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "id": "fX6nUR8OvLgu", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 199 - }, - "outputId": "7d9a9d51-ebe3-4d72-a949-9b5854e544c6" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stderr", - "text": [ - "/usr/local/lib/python3.10/dist-packages/transformers/pipelines/text_classification.py:104: UserWarning: `return_all_scores` is now deprecated, if want a similar functionality use `top_k=None` instead of `return_all_scores=True` or `top_k=1` instead of `return_all_scores=False`.\n", - " warnings.warn(\n" - ] - }, - { - "output_type": "execute_result", - "data": { - "text/plain": [ - " Query Output Branch Class\n", - "0 What's the answer? LABEL_1 neutral\n", - "1 Would you be so lovely to tell me the answer? LABEL_2 positive\n", - "2 Can you give me the damn right answer for once?? LABEL_0 negative" - ], - "text/html": [ - "\n", - "
\n", - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
QueryOutput BranchClass
0What's the answer?LABEL_1neutral
1Would you be so lovely to tell me the answer?LABEL_2positive
2Can you give me the damn right answer for once??LABEL_0negative
\n", - "
\n", - "
\n", - "\n", - "
\n", - " \n", - "\n", - " \n", - "\n", - " \n", - "
\n", - "\n", - "\n", - "
\n", - " \n", - "\n", - "\n", - "\n", - " \n", - "
\n", - "\n", - "
\n", - "
\n" - ], - "application/vnd.google.colaboratory.intrinsic+json": { - "type": "dataframe", - "summary": "{\n \"name\": \"pd\",\n \"rows\": 3,\n \"fields\": [\n {\n \"column\": \"Query\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 3,\n \"samples\": [\n \"What's the answer?\",\n \"Would you be so lovely to tell me the answer?\",\n \"Can you give me the damn right answer for once??\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Output Branch\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 3,\n \"samples\": [\n \"LABEL_1\",\n \"LABEL_2\",\n \"LABEL_0\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Class\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 3,\n \"samples\": [\n \"neutral\",\n \"positive\",\n \"negative\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}" - } - }, - "metadata": {}, - "execution_count": 10 - } - ], - "source": [ - "sent_results = {\"Query\": [], \"Output Branch\": [], \"Class\": []}\n", - "\n", - "for query in queries:\n", - " result = text_router.run(text=query)\n", - " sent_results[\"Query\"].append(query)\n", - " sent_results[\"Output Branch\"].append(next(iter(result)))\n", - " sent_results[\"Class\"].append({\"LABEL_0\": \"negative\", \"LABEL_1\": \"neutral\", \"LABEL_2\":\"positive\"}.get(next(iter(result)), \"Unknown\"))\n", - "\n", - "pd.DataFrame.from_dict(sent_results)" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "==============================\n", + "QUESTION QUERY RESULTS\n", + "==============================\n", + "{'document_joiner': {'documents': [Document(id=4c82325818ccd91af8d68fec37108ce7a93696392f315bd0497ad3a8903d0b45, content: 'The Masonic House of the Temple of the Scottish Rite, Washington, DC, designed by John Russell Pope,...', meta: {'url': 'https://en.wikipedia.org/wiki/Mausoleum_at_Halicarnassus', '_split_id': 18}, score: 8.192663165691801, embedding: vector of size 384), Document(id=4a988f268c10bbb6af9a18063a14460b7e0126c7ed1befb2be17c9cbbc4bb064, content: 'The earliest pharaonic name of seal impressions is that of Khufu, the latest of Pepi II. Worker graf...', meta: {'url': 'https://en.wikipedia.org/wiki/Great_Pyramid_of_Giza', '_split_id': 4}, score: 6.652366985087608, embedding: vector of size 384), Document(id=9c1d7b92058a18bd101c037c87519e9225983c1dbb9386d51412895d5101d096, content: '[68]\n", + "The most notable account of this legend was given by Al-Masudi (896–956) in his Akbar al-zaman,...', meta: {'url': 'https://en.wikipedia.org/wiki/Great_Pyramid_of_Giza', '_split_id': 19}, score: 5.365566894478934, embedding: vector of size 384), Document(id=1ac6d4aedca0f952d87a1cfd4d4bde946d54f103a8f802419671d9ad2e26f3bc, content: '[21] However, the gardens were said to still exist at the time that later writers described them, an...', meta: {'url': 'https://en.wikipedia.org/wiki/Hanging_Gardens_of_Babylon', '_split_id': 5}, score: 5.279086706756248, embedding: vector of size 384), Document(id=975a20d36be68ebf6c7c7a37012dd0deff29797201dddb21e1b3a25c29b168e2, content: 'Modern historians have pointed out that two years would not be enough time to decorate and build suc...', meta: {'url': 'https://en.wikipedia.org/wiki/Mausoleum_at_Halicarnassus', '_split_id': 5}, score: 4.983734342490586, embedding: vector of size 384), Document(id=5d02e37e955d05869cad244b01e9c458c0676a736390b04f347390aa6d8236f7, content: 'The passage is 2 cubits (1.0 m; 3.4 ft) wide and 1.17 m (3.8 ft) high for most of its length, but ne...', meta: {'url': 'https://en.wikipedia.org/wiki/Great_Pyramid_of_Giza', '_split_id': 45}, score: 4.9689671066012755, embedding: vector of size 384), Document(id=06ce9e11253c0c619159c6bd9c8f8e711d7a841025efc696e07b74000c5f750a, content: 'The remains were described briefly by Strabo (64 or 63 BC – c. 24 AD), in his work Geography (Book X...', meta: {'url': 'https://en.wikipedia.org/wiki/Colossus_of_Rhodes', '_split_id': 6}, score: 4.956812213009003, embedding: vector of size 384), Document(id=8898fdd9ba5e7049fcec94ae5e9f172337de969977ccbfb1d5e2b23baf964d15, content: 'Only Josephus names Nebuchadnezzar as the king who built the gardens; although Nebuchadnezzar left m...', meta: {'url': 'https://en.wikipedia.org/wiki/Hanging_Gardens_of_Babylon', '_split_id': 7}, score: 4.950459961177993, embedding: vector of size 384), Document(id=e42388e23323dad64d9cd5dd5e93fa1391d93be3dc161236d789e2eb7de99546, content: 'These were entrusted to a boat builder, Haj Ahmed Yusuf, who worked out how the pieces fit together....', meta: {'url': 'https://en.wikipedia.org/wiki/Great_Pyramid_of_Giza', '_split_id': 60}, score: 4.917705216995133, embedding: vector of size 384), Document(id=417cb72c998c669851f4cfc8b4a2ba75c7ab8fd2e880f53ea10e6c75d8e0a6b2, content: 'Further, he describes a method discovered by Thales of Miletus for ascertaining the pyramid's height...', meta: {'url': 'https://en.wikipedia.org/wiki/Great_Pyramid_of_Giza', '_split_id': 17}, score: 4.9033381526508935, embedding: vector of size 384)]}}\n" + ] }, { - "cell_type": "markdown", - "metadata": { - "id": "-6R8jfJVvLgu" - }, - "source": [ - "### Zero-Shot Classification with TransformersZeroShotTextRouter\n", - "TransformersZeroShotTextRouter let's you perform zero-shot classification by providing a suitable base transformer model and **defining** the classes the model should predict.\n", - "\n", - "First, initialize a TransformersZeroShotTextRouter with some custom category labels. By default, it uses the base size zero shot classification model [MoritzLaurer/deberta-v3-base-zeroshot-v1.1-all-33](https://huggingface.co/MoritzLaurer/deberta-v3-base-zeroshot-v1.1-all-33). You can switch to the larger model [MoritzLaurer/deberta-v3-large-zeroshot-v2.0](https://huggingface.co/MoritzLaurer/deberta-v3-large-zeroshot-v2.0) and get even better results by using `TransformersZeroShotTextRouter(model=\"MoritzLaurer/deberta-v3-large-zeroshot-v2.0\")`.\n", - "\n", - "Let's look at an example. You may be interested in whether the user query is related to music or cinema. In this case, the `labels` parameter is a list containing the candidate classes and the output branches of the component are named accordingly." - ] + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.10/dist-packages/transformers/pipelines/text_classification.py:104: UserWarning: `return_all_scores` is now deprecated, if want a similar functionality use `top_k=None` instead of `return_all_scores=True` or `top_k=1` instead of `return_all_scores=False`.\n", + " warnings.warn(\n" + ] }, { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "xO7sr516vLgu" + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "93e5e993baf346b39205fab6d988da84", + "version_major": 2, + "version_minor": 0 }, - "outputs": [], - "source": [ - "from haystack.components.routers import TransformersZeroShotTextRouter\n", - "\n", - "text_router = TransformersZeroShotTextRouter(labels=[\"music\", \"cinema\"])\n", - "text_router.warm_up()" + "text/plain": [ + "Batches: 0%| | 0/1 [00:00\n", - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
QueryOutput Branch
0In which films does John Travolta appear?cinema
1What is the Rolling Stones first album?music
2Who was Sergio Leone?cinema
\n", - "
\n", - "
\n", - "\n", - "
\n", - " \n", - "\n", - " \n", - "\n", - " \n", - "
\n", - "\n", - "\n", - "
\n", - " \n", - "\n", - "\n", - "\n", - " \n", - "
\n", - "\n", - "
\n", - " \n" - ], - "application/vnd.google.colaboratory.intrinsic+json": { - "type": "dataframe", - "summary": "{\n \"name\": \"pd\",\n \"rows\": 3,\n \"fields\": [\n {\n \"column\": \"Query\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 3,\n \"samples\": [\n \"In which films does John Travolta appear?\",\n \"What is the Rolling Stones first album?\",\n \"Who was Sergio Leone?\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Output Branch\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 2,\n \"samples\": [\n \"music\",\n \"cinema\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}" - } - }, - "metadata": {}, - "execution_count": 13 - } - ], - "source": [ - "sent_results = {\"Query\": [], \"Output Branch\": []}\n", - "\n", - "for query in queries:\n", - " result = text_router.run(text=query)\n", - " sent_results[\"Query\"].append(query)\n", - " sent_results[\"Output Branch\"].append(next(iter(result)))\n", - "\n", - "pd.DataFrame.from_dict(sent_results)" + "data": { + "text/plain": [ + "\n", + "🚅 Components\n", + " - bm25_retriever_0: InMemoryBM25Retriever\n", + " - bm25_retriever_1: InMemoryBM25Retriever\n", + " - text_router: TransformersTextRouter\n", + " - reader: ExtractiveReader\n", + "🛤️ Connections\n", + " - bm25_retriever_0.documents -> reader.documents (List[Document])\n", + " - text_router.LABEL_0 -> bm25_retriever_0.query (str)\n", + " - text_router.LABEL_1 -> bm25_retriever_1.query (str)" ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from haystack.components.readers import ExtractiveReader\n", + "\n", + "query_classification_pipeline = Pipeline()\n", + "query_classification_pipeline.add_component(\"bm25_retriever_0\", InMemoryBM25Retriever(document_store))\n", + "query_classification_pipeline.add_component(\"bm25_retriever_1\", InMemoryBM25Retriever(document_store))\n", + "query_classification_pipeline.add_component(\n", + " \"text_router\", TransformersTextRouter(model=\"shahrukhx01/question-vs-statement-classifier\")\n", + ")\n", + "query_classification_pipeline.add_component(\"reader\", ExtractiveReader())\n", + "\n", + "query_classification_pipeline.connect(\"text_router.LABEL_0\", \"bm25_retriever_0\")\n", + "query_classification_pipeline.connect(\"bm25_retriever_0\", \"reader\")\n", + "query_classification_pipeline.connect(\"text_router.LABEL_1\", \"bm25_retriever_1\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "QU1B6JQEDrol" + }, + "source": [ + "### 2) Run the Pipeline\n", + "And here are the results of this pipeline: with a question query like \"Who is the father of Arya Stark?\", you obtain answers from a Reader, and with a statement query like \"Arya Stark was the daughter of a Lord\", you just obtain documents from a Retriever." + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 493, + "referenced_widgets": [ + "86d7ab02d471454ebb7560adf2af3734", + "c07495fe464e4793af8d7d0ab6cbe68b", + "f55a41788e4a49c9844b4b02274f1c35", + "9f777bed57e5486ab6d833b65155c9ec", + "96dc7d5082474169a8ed0fe90fda29be", + "bcd896e543794c2fb0df897931cedcf6", + "d58b712e8dda486d8a3def5a2e46c8fa", + "8e0bb920cc15493db671961d76985397", + "6072b48ab7fb4d0fac14902eb4f609bd", + "bc2d103625d547c78a7e14011710c5c3", + "53aa58c50eea49e1874ccdbca72af8ec", + "c8c7816cb48f46ca8b419f8c7efb8f5e", + "b400314ef3644290ac4f4e86ef3bf17b", + "93f4cadf8aad48d19d60d4861386926b", + "52e9d9c6c4804b5392d413e2b1f70203", + "dae3785870ff4d3b872798cc76ce6dcd", + "6ac3897894094303979a5a675a0a0c4c", + "f2cbfde5e1044ed388a94defa478d5a5", + "d88cb0502ba649d9b01fdbcd3d059b11", + "5b0ee86da7254efebee0292ffcaaa9fb", + "b204177f854b45768b270e3c69e6a266", + "277b0f8022654e8e957845cd9ee57493", + "e0f6aea68492445b976f2dde5c87cf16", + "435532e97b19419fb0240cc9c2f2a6dc", + "1c76d5b3076746c2a2d78b5851f7bfa2", + "bcb3c70d00fd43b6ac66ace3153a6d7e", + "5627acc9660a496d84d65b1f8c280bdb", + "644ebc3128f24e6daae1856e142c0386", + "b6354198917840c4bc942b9cd4667f93", + "89d9afef1c344a72ba188d1ba621d1af", + "00debc7e71fc4d6bb0af6d8a6f25c0f7", + "e9ee9ef7c4724e6d8ea20bdd6d6ef080", + "5f95340e313746a1b1c7a6f27abb4600", + "67001f19b6d5442f8f77f7bb0695a8ea", + "ba7e6ff00aca4f0ab5c44a3ccf65bb3a", + "4c3f1c744fab4ade95eba8125e116d4d", + "594e4a67550549d7ab6e4ceb70bd3ee8", + "6a3f9587f7ed4f7fa207542f9351f905", + "4a8706a789d5479493bf122020513e88", + "609d36079b144c16bc6d6bf49d0062f7", + "fd00db3af05c486b8c8d04a60366be51", + "bc7c67380dfc45668f9cd430e8516000", + "c978c043a67a40dd9497143f98b8ac87", + "956c5d4526b643f7bb5592d4477f82f2", + "517fab9a3db348dc8964fbf921e9f61f", + "5b1dd24a09b14c8eaa5c7601ebf3705e", + "8fe73a8d53bb403c9c2eb35efcd536fd", + "dad54c47738d4afea261cc1cee2df8fd", + "47d0530844184daf9bc90c68ddb4f9fa", + "9fbeeebbbbe347f7a9572e5fb234fcfd", + "300a3c39844d40588ab543b85f2f29c4", + "e7291573b13542c7a0d80795a974c4d1", + "df66f8d6b8d6494782143e1776cbca7f", + "c9dc77a6e6c64c328e49a0f2bea925b1", + "b4da651ee89f48c99cbc7b7bbaf7c884", + "9033ee66e6da494d85ae616fc0efba1e", + "9f2a6b2692ab45c4900842b93c3a4742", + "5020e9035ba8423ea51109a23a95bfd2", + "2d7cfb3c17a440b0988d5b9a71e9c634", + "266bc71b03824c42b0f93f98dd299784", + "63698c2fa2c645fc8df856e1b7c914bb", + "9270418511e945f2863b2866b81a2168", + "ffb7ed4a6f984068a46330e3bae16027", + "8ce0c21b77dd4bcea15a730eb6f128c0", + "fe1f34d442d14f0dbf428c86150f59aa", + "e956b75c71bb480ba366220c6eea62f6", + "d2d3a829d57f40a5af01e74bf9919cfa", + "9c2c4061fe7247c59eea7dddb37de56a", + "0e1cffb58c94491ca7246ffb3d9e0657", + "2fc62a3826dc4c7e95145ac24cfdb40d", + "b1c20311cec84d528f8050f5de69abc3", + "5dc9b503c5094dce8849694e2a6dfaeb", + "27ab028d35804b8fa1a8023b5e8525e6", + "0328a80806e84a60821b74200f6581f5", + "bf3bec07aa4946fc8fa02359098c0cc3", + "f3e3192dcead450d89fd7dd2a57a8a3c", + "fb70857ca8e44d399ae0525f099ca230" + ] }, + "id": "HIjgs5k7C6CN", + "outputId": "f3f94168-433e-4955-89b2-ae33d245b084" + }, + "outputs": [ { - "cell_type": "markdown", - "source": [ - "Similar to the previous example, we can use zero-shot text classification to group questions into \"Game of Thrones\", \"Star Wars\" and \"Lord of the Rings\" related question. The number of labels is up to you!" - ], - "metadata": { - "id": "ZuGnJLMKSeNm" - } + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "86d7ab02d471454ebb7560adf2af3734", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "config.json: 0%| | 0.00/729 [00:00\n", - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
QueryOutput Branch
0Who was the father of Arya StarkGame of Thrones
1Who was the father of Luke SkywalkerStar Wars
2Who was the father of Frodo BagginsLord of the Rings
\n", - "
\n", - "
\n", - "\n", - "
\n", - " \n", - "\n", - " \n", - "\n", - " \n", - "
\n", - "\n", - "\n", - "
\n", - " \n", - "\n", - "\n", - "\n", - " \n", - "
\n", - "\n", - "
\n", - " \n" - ], - "application/vnd.google.colaboratory.intrinsic+json": { - "type": "dataframe", - "summary": "{\n \"name\": \"pd\",\n \"rows\": 3,\n \"fields\": [\n {\n \"column\": \"Query\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 3,\n \"samples\": [\n \"Who was the father of Arya Stark\",\n \"Who was the father of Luke Skywalker\",\n \"Who was the father of Frodo Baggins\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Output Branch\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 3,\n \"samples\": [\n \"Game of Thrones\",\n \"Star Wars\",\n \"Lord of the Rings\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}" - } - }, - "metadata": {}, - "execution_count": 14 - } - ], - "source": [ - "from haystack.components.routers import TransformersZeroShotTextRouter\n", - "\n", - "text_router = TransformersZeroShotTextRouter(labels=[\"Game of Thrones\", \"Star Wars\", \"Lord of the Rings\"])\n", - "text_router.warm_up()\n", - "\n", - "queries = [\n", - " \"Who was the father of Arya Stark\", # Game of Thrones\n", - " \"Who was the father of Luke Skywalker\", # Star Wars\n", - " \"Who was the father of Frodo Baggins\", # Lord of the Rings\n", - "]\n", - "\n", - "results = {\"Query\": [], \"Output Branch\": []}\n", - "\n", - "for query in queries:\n", - " result = text_router.run(text=query)\n", - " results[\"Query\"].append(query)\n", - " results[\"Output Branch\"].append(next(iter(result)))\n", - "\n", - "pd.DataFrame.from_dict(results)" + "text/plain": [ + "model.safetensors: 0%| | 0.00/496M [00:00 `InMemoryDocumentStore` is the simplest DocumentStore to get started with. It requires no external dependencies and it's a good option for smaller projects and debugging. But it doesn't scale up so well to larger Document collections, so it's not a good choice for production systems. To learn more about the different types of external databases that Haystack supports, see [DocumentStore Integrations](https://haystack.deepset.ai/integrations?type=Document+Store)." - ], - "metadata": { - "id": "dqAtAD3mVTgK" - } + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "==============================\n", + "QUESTION QUERY RESULTS\n", + "==============================\n", + "{'bm25_retriever_1': {'documents': [Document(id=4c82325818ccd91af8d68fec37108ce7a93696392f315bd0497ad3a8903d0b45, content: 'The Masonic House of the Temple of the Scottish Rite, Washington, DC, designed by John Russell Pope,...', meta: {'url': 'https://en.wikipedia.org/wiki/Mausoleum_at_Halicarnassus', '_split_id': 18}, score: 8.192663165691801, embedding: vector of size 384), Document(id=4a988f268c10bbb6af9a18063a14460b7e0126c7ed1befb2be17c9cbbc4bb064, content: 'The earliest pharaonic name of seal impressions is that of Khufu, the latest of Pepi II. Worker graf...', meta: {'url': 'https://en.wikipedia.org/wiki/Great_Pyramid_of_Giza', '_split_id': 4}, score: 6.652366985087608, embedding: vector of size 384), Document(id=9c1d7b92058a18bd101c037c87519e9225983c1dbb9386d51412895d5101d096, content: '[68]\n", + "The most notable account of this legend was given by Al-Masudi (896–956) in his Akbar al-zaman,...', meta: {'url': 'https://en.wikipedia.org/wiki/Great_Pyramid_of_Giza', '_split_id': 19}, score: 5.365566894478934, embedding: vector of size 384), Document(id=1ac6d4aedca0f952d87a1cfd4d4bde946d54f103a8f802419671d9ad2e26f3bc, content: '[21] However, the gardens were said to still exist at the time that later writers described them, an...', meta: {'url': 'https://en.wikipedia.org/wiki/Hanging_Gardens_of_Babylon', '_split_id': 5}, score: 5.279086706756248, embedding: vector of size 384), Document(id=975a20d36be68ebf6c7c7a37012dd0deff29797201dddb21e1b3a25c29b168e2, content: 'Modern historians have pointed out that two years would not be enough time to decorate and build suc...', meta: {'url': 'https://en.wikipedia.org/wiki/Mausoleum_at_Halicarnassus', '_split_id': 5}, score: 4.983734342490586, embedding: vector of size 384), Document(id=5d02e37e955d05869cad244b01e9c458c0676a736390b04f347390aa6d8236f7, content: 'The passage is 2 cubits (1.0 m; 3.4 ft) wide and 1.17 m (3.8 ft) high for most of its length, but ne...', meta: {'url': 'https://en.wikipedia.org/wiki/Great_Pyramid_of_Giza', '_split_id': 45}, score: 4.9689671066012755, embedding: vector of size 384), Document(id=06ce9e11253c0c619159c6bd9c8f8e711d7a841025efc696e07b74000c5f750a, content: 'The remains were described briefly by Strabo (64 or 63 BC – c. 24 AD), in his work Geography (Book X...', meta: {'url': 'https://en.wikipedia.org/wiki/Colossus_of_Rhodes', '_split_id': 6}, score: 4.956812213009003, embedding: vector of size 384), Document(id=8898fdd9ba5e7049fcec94ae5e9f172337de969977ccbfb1d5e2b23baf964d15, content: 'Only Josephus names Nebuchadnezzar as the king who built the gardens; although Nebuchadnezzar left m...', meta: {'url': 'https://en.wikipedia.org/wiki/Hanging_Gardens_of_Babylon', '_split_id': 7}, score: 4.950459961177993, embedding: vector of size 384), Document(id=e42388e23323dad64d9cd5dd5e93fa1391d93be3dc161236d789e2eb7de99546, content: 'These were entrusted to a boat builder, Haj Ahmed Yusuf, who worked out how the pieces fit together....', meta: {'url': 'https://en.wikipedia.org/wiki/Great_Pyramid_of_Giza', '_split_id': 60}, score: 4.917705216995133, embedding: vector of size 384), Document(id=417cb72c998c669851f4cfc8b4a2ba75c7ab8fd2e880f53ea10e6c75d8e0a6b2, content: 'Further, he describes a method discovered by Thales of Miletus for ascertaining the pyramid's height...', meta: {'url': 'https://en.wikipedia.org/wiki/Great_Pyramid_of_Giza', '_split_id': 17}, score: 4.9033381526508935, embedding: vector of size 384)]}}\n", + "\n", + "\n", + "==============================\n", + "KEYWORD QUERY RESULTS\n", + "==============================\n", + "{'reader': {'answers': [ExtractedAnswer(query='Arya Stark was the daughter of a Lord', score=0.3654985725879669, data='King of Upper and Lower Egypt Khufu', document=Document(id=4a988f268c10bbb6af9a18063a14460b7e0126c7ed1befb2be17c9cbbc4bb064, content: 'The earliest pharaonic name of seal impressions is that of Khufu, the latest of Pepi II. Worker graf...', meta: {'url': 'https://en.wikipedia.org/wiki/Great_Pyramid_of_Giza', '_split_id': 4}, score: 8.527383694416635, embedding: vector of size 384), context=None, document_offset=ExtractedAnswer.Span(start=518, end=553), context_offset=None, meta={}), ExtractedAnswer(query='Arya Stark was the daughter of a Lord', score=0.3226390779018402, data=\"Phidias' workshop at Olympia\\nPhidias' workshop\", document=Document(id=60469411be57782b4a518ee430379e5235c5dff6f279fe232fae0adfb840644d, content: 'The 11th-century Byzantine historian Georgios Kedrenos records a tradition that it was carried off t...', meta: {'url': 'https://en.wikipedia.org/wiki/Statue_of_Zeus_at_Olympia', '_split_id': 5}, score: 8.40941628722177, embedding: vector of size 384), context=None, document_offset=ExtractedAnswer.Span(start=698, end=744), context_offset=None, meta={}), ExtractedAnswer(query='Arya Stark was the daughter of a Lord', score=0.29660001397132874, data='\\nPhoto', document=Document(id=60469411be57782b4a518ee430379e5235c5dff6f279fe232fae0adfb840644d, content: 'The 11th-century Byzantine historian Georgios Kedrenos records a tradition that it was carried off t...', meta: {'url': 'https://en.wikipedia.org/wiki/Statue_of_Zeus_at_Olympia', '_split_id': 5}, score: 8.40941628722177, embedding: vector of size 384), context=None, document_offset=ExtractedAnswer.Span(start=681, end=687), context_offset=None, meta={}), ExtractedAnswer(query='Arya Stark was the daughter of a Lord', score=0.2902381718158722, data='\\nConstruction', document=Document(id=055b1454beddf2064ddfd8079da93ddffd5f9a0dfa25fb5321835b914312c32b, content: 'In addition to measuring the structure, alongside the other pyramids at Giza, al-Baghdadi also write...', meta: {'url': 'https://en.wikipedia.org/wiki/Great_Pyramid_of_Giza', '_split_id': 21}, score: 5.86912829515654, embedding: vector of size 384), context=None, document_offset=ExtractedAnswer.Span(start=892, end=905), context_offset=None, meta={}), ExtractedAnswer(query='Arya Stark was the daughter of a Lord', score=0.2845076620578766, data='Lighthouse of Alexandria by Philip Galle; 1572, Rijksmuseum', document=Document(id=09af7baea84fbe125b3321c9fdb851cbe9f3fdc54d5e4f21b2f6e63134882d24, content: 'The Ras el-Tin promontory, where Ras el-Tin Palace was built in the 19th century, represents all tha...', meta: {'url': 'https://en.wikipedia.org/wiki/Lighthouse_of_Alexandria', '_split_id': 2}, score: 5.869676649867554, embedding: vector of size 384), context=None, document_offset=ExtractedAnswer.Span(start=245, end=304), context_offset=None, meta={}), ExtractedAnswer(query='Arya Stark was the daughter of a Lord', score=0.27953213453292847, data='Preparation of the site\\nA hillock forms the base on which the pyramid stands. It was cut back into steps and only a strip around the perimeter was leveled,[78] which has been measured to be horizontal and flat to within 21 millimetres (0.', document=Document(id=055b1454beddf2064ddfd8079da93ddffd5f9a0dfa25fb5321835b914312c32b, content: 'In addition to measuring the structure, alongside the other pyramids at Giza, al-Baghdadi also write...', meta: {'url': 'https://en.wikipedia.org/wiki/Great_Pyramid_of_Giza', '_split_id': 21}, score: 5.86912829515654, embedding: vector of size 384), context=None, document_offset=ExtractedAnswer.Span(start=906, end=1144), context_offset=None, meta={}), ExtractedAnswer(query='Arya Stark was the daughter of a Lord', score=0.27728188037872314, data='Lord Stratford de Redcliffe', document=Document(id=72d9ba5c859a4737ea03759f89a6d4fb50cab7f04061de6c33e1835ef7f8eedf, content: 'Because of this, Fergusson concluded that the building was ruined, probably by an earthquake, betwee...', meta: {'url': 'https://en.wikipedia.org/wiki/Mausoleum_at_Halicarnassus', '_split_id': 6}, score: 8.444858211584728, embedding: vector of size 384), context=None, document_offset=ExtractedAnswer.Span(start=622, end=649), context_offset=None, meta={}), ExtractedAnswer(query='Arya Stark was the daughter of a Lord', score=0.26334038376808167, data='4 horse chariots of marble. The building was accented with both sculptural friezes and free standing figures. \"The free standing figures were arranged on 5 or 6 different levels.\"[9]\\nWe are now able to justify that Pliny\\'s knowledge came from a work written by the architect. It is clear that Pliny did not grasp the design of the mausoleum fully which creates problems in recreating the structure. He does state many facts which help the reader recreate pieces of the puzzle. Other writings by Pausanias, Strabo, and Vitruvius also help us to gather more information about the Mausoleum.[21]\\nAccording to Pliny, the mausoleum was 19 metres (63\\xa0ft) north and south, shorter on other fronts, 125 metres (411\\xa0ft) perimeter, and 25\\xa0cubits (11.', document=Document(id=0b3ec68a3754c399d2160c93c36ea172c318edad9310828c7d056392956f884f, content: 'The building was rectangular, not square, surrounded by a colonnade of thirty-six columns. There was...', meta: {'url': 'https://en.wikipedia.org/wiki/Mausoleum_at_Halicarnassus', '_split_id': 9}, score: 5.870291493549832, embedding: vector of size 384), context=None, document_offset=ExtractedAnswer.Span(start=191, end=931), context_offset=None, meta={}), ExtractedAnswer(query='Arya Stark was the daughter of a Lord', score=0.2617728114128113, data='Antipater of Sidon identified it as one of his Seven Wonders of the Ancient World. It was destroyed by successive earthquakes from the 12th to the 15th century;[6][7][8', document=Document(id=9b6d3d0d9599886ec38a2476e741988900133e6d57056f4d4718976ab7bc8d87, content: 'The Mausoleum at Halicarnassus or Tomb of Mausolus[a] (Ancient Greek: Μαυσωλεῖον τῆς Ἁλικαρνασσοῦ; T...', meta: {'url': 'https://en.wikipedia.org/wiki/Mausoleum_at_Halicarnassus', '_split_id': 0}, score: 5.870202729262424, embedding: vector of size 384), context=None, document_offset=ExtractedAnswer.Span(start=926, end=1094), context_offset=None, meta={}), ExtractedAnswer(query='Arya Stark was the daughter of a Lord', score=0.252872496843338, data='Mausolus[a] (Ancient Greek: Μαυσωλεῖον τῆς Ἁλικαρνασσοῦ; Turkish: Halikarnas Mozolesi) was a tomb built between 353 and 350\\xa0BC in Halicarnassus (present Bodrum, Turkey) for Mausolus, an Anatolian from Caria and a satrap in the Achaemenid Empire, and his sister-wife Artemisia II of Caria', document=Document(id=9b6d3d0d9599886ec38a2476e741988900133e6d57056f4d4718976ab7bc8d87, content: 'The Mausoleum at Halicarnassus or Tomb of Mausolus[a] (Ancient Greek: Μαυσωλεῖον τῆς Ἁλικαρνασσοῦ; T...', meta: {'url': 'https://en.wikipedia.org/wiki/Mausoleum_at_Halicarnassus', '_split_id': 0}, score: 5.870202729262424, embedding: vector of size 384), context=None, document_offset=ExtractedAnswer.Span(start=42, end=329), context_offset=None, meta={}), ExtractedAnswer(query='Arya Stark was the daughter of a Lord', score=0.2474311888217926, data='Artemis Tapınağı), also known as the Temple of Diana, was a Greek temple dedicated to an ancient, local form of the goddess Artemis (identified with Diana, a Roman goddess). It was located in Ephesus (near the modern town of Selçuk in present-day Turkey). By 401 AD it had been ruined or destroyed.[1] Only foundations and fragments of the last temple remain at the site.\\nThe earliest version of the temple (a Bronze Age temenos) antedated the Ionic immigration by many years. Callimachus, in his Hymn to Artemis, attributed it to the Amazons. In the 7th century BC, it was destroyed by a flood.\\nIts reconstruction, in more grandiose form, began around 550 BC, under Chersiphron, the Cretan architect, and his son Metagenes. The project was funded by Croesus of Lydia, and took 10 years to complete. This version of the temple was destroyed in 356 BC by an arsonist.\\n', document=Document(id=e6244310d1a0686d06b52b93c1d96972a8d01943be7735b6593a0ffadd091b3c, content: 'The Temple of Artemis or Artemision (Greek: Ἀρτεμίσιον; Turkish: Artemis Tapınağı), also known as th...', meta: {'url': 'https://en.wikipedia.org/wiki/Temple_of_Artemis', '_split_id': 0}, score: 5.868186231095871, embedding: vector of size 384), context=None, document_offset=ExtractedAnswer.Span(start=65, end=932), context_offset=None, meta={}), ExtractedAnswer(query='Arya Stark was the daughter of a Lord', score=0.24727863073349, data='Sennacherib was proud of the technologies he had employed and describes them in some detail on his inscriptions. At the headwater of Bavian (Khinnis)[39] his inscription mentions automatic sluice gates. An enormous aqueduct crossing the valley at Jerwan was constructed of over two million dressed stones. It used stone arches and waterproof cement.[40] On it is written:\\n\\nSennacherib king of the world king of Assyria', document=Document(id=f736d7b87b1e0b18d32a823cacc82df7773fd3e6437b6d3e6b1dfd9b49dd80e6, content: 'Sennacherib was proud of the technologies he had employed and describes them in some detail on his i...', meta: {'url': 'https://en.wikipedia.org/wiki/Hanging_Gardens_of_Babylon', '_split_id': 9}, score: 5.862567822134554, embedding: vector of size 384), context=None, document_offset=ExtractedAnswer.Span(start=0, end=418), context_offset=None, meta={}), ExtractedAnswer(query='Arya Stark was the daughter of a Lord', score=0.24462197721004486, data='Sennacherib claimed that he had built a \"Wonder for all Peoples\", and said he was the first to deploy a new casting technique in place of the \"lost-wax\" process for his monumental (30 tonne) bronze castings. He was able to bring the water into his garden at a high level because it was sourced from further up in the mountains, and he then raised the water even higher by deploying his new water screws', document=Document(id=f736d7b87b1e0b18d32a823cacc82df7773fd3e6437b6d3e6b1dfd9b49dd80e6, content: 'Sennacherib was proud of the technologies he had employed and describes them in some detail on his i...', meta: {'url': 'https://en.wikipedia.org/wiki/Hanging_Gardens_of_Babylon', '_split_id': 9}, score: 5.862567822134554, embedding: vector of size 384), context=None, document_offset=ExtractedAnswer.Span(start=639, end=1041), context_offset=None, meta={}), ExtractedAnswer(query='Arya Stark was the daughter of a Lord', score=0.23422574996948242, data='Mausolus before his death or continued by the next leaders', document=Document(id=975a20d36be68ebf6c7c7a37012dd0deff29797201dddb21e1b3a25c29b168e2, content: 'Modern historians have pointed out that two years would not be enough time to decorate and build suc...', meta: {'url': 'https://en.wikipedia.org/wiki/Mausoleum_at_Halicarnassus', '_split_id': 5}, score: 5.887070537721882, embedding: vector of size 384), context=None, document_offset=ExtractedAnswer.Span(start=195, end=253), context_offset=None, meta={}), ExtractedAnswer(query='Arya Stark was the daughter of a Lord', score=0.22647136449813843, data='Eustathius, writing in the 12th century on his commentary of the Iliad, says \"it was and is a wonder', document=Document(id=975a20d36be68ebf6c7c7a37012dd0deff29797201dddb21e1b3a25c29b168e2, content: 'Modern historians have pointed out that two years would not be enough time to decorate and build suc...', meta: {'url': 'https://en.wikipedia.org/wiki/Mausoleum_at_Halicarnassus', '_split_id': 5}, score: 5.887070537721882, embedding: vector of size 384), context=None, document_offset=ExtractedAnswer.Span(start=900, end=1000), context_offset=None, meta={}), ExtractedAnswer(query='Arya Stark was the daughter of a Lord', score=0.008232464457383926, data=None, document=None, context=None, document_offset=None, context_offset=None, meta={})]}}\n" + ] + } + ], + "source": [ + "# Useful for framing headers\n", + "equal_line = \"=\" * 30\n", + "\n", + "# Run the retriever + reader on the question query\n", + "query = \"Who is the father of Arya Stark?\"\n", + "res_1 = query_classification_pipeline.run({\"text_router\": {\"text\": query}, \"reader\": {\"query\": query}})\n", + "print(f\"\\n\\n{equal_line}\\nQUESTION QUERY RESULTS\\n{equal_line}\")\n", + "print(res_1)\n", + "\n", + "# Run only the retriever on the statement query\n", + "query = \"Arya Stark was the daughter of a Lord\"\n", + "res_2 = query_classification_pipeline.run({\"text_router\": {\"text\": query}, \"reader\": {\"query\": query}})\n", + "print(f\"\\n\\n{equal_line}\\nKEYWORD QUERY RESULTS\\n{equal_line}\")\n", + "print(res_2)" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3.10.6 64-bit", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.6 (main, Aug 11 2022, 13:36:31) [Clang 13.1.6 (clang-1316.0.21.2.5)]" + }, + "vscode": { + "interpreter": { + "hash": "bda33b16be7e844498c7c2d368d72665b4f1d165582b9547ed22a0249a29ca2e" + } + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "00debc7e71fc4d6bb0af6d8a6f25c0f7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } }, - { - "cell_type": "markdown", - "source": [ - "The DocumentStore is now ready. Now it's time to fill it with some Documents." + "011a4dc918c54bc19e48e7f0c99c95f2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "0328a80806e84a60821b74200f6581f5": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "0c4a42c7897d4da1b3e7b62007196854": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_dcf6276e44cd491c912d1b09502fc91b", + "placeholder": "​", + "style": "IPY_MODEL_63dcf941fcc94f47bb7d590a5e2f54b8", + "value": "Batches: 100%" + } + }, + "0c913d57bbe54ac59702e61c4ba753a9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_20177ae3a9f14622a1fef3196159cb44", + "IPY_MODEL_b24e12ce7fba4841b76ed518d4659356", + "IPY_MODEL_770d46aa7dca4ad497d3d5c64ce5147e" ], - "metadata": { - "id": "ZpdVE3V8VX6u" - } + "layout": "IPY_MODEL_f9d574e591e64ed8b764103d11c437c0" + } }, - { - "cell_type": "markdown", - "source": [ - "### Fetch the Data\n", - "\n", - "You'll use the Wikipedia pages of [Seven Wonders of the Ancient World](https://en.wikipedia.org/wiki/Wonders_of_the_World) as Documents. We preprocessed the data and uploaded to a Hugging Face Space: [Seven Wonders](https://huggingface.co/datasets/bilgeyucel/seven-wonders). Thus, you don't need to perform any additional cleaning or splitting.\n", - "\n", - "Fetch the data and convert it into Haystack Documents:" + "0e1cffb58c94491ca7246ffb3d9e0657": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0328a80806e84a60821b74200f6581f5", + "max": 239, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_bf3bec07aa4946fc8fa02359098c0cc3", + "value": 239 + } + }, + "138d4dda99b4450a88a311faea032010": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_fd822ca15aab4929a8a61adfce6e6479", + "placeholder": "​", + "style": "IPY_MODEL_4f02f9ae1623496f8754406eb24527b1", + "value": "tokenizer_config.json: 100%" + } + }, + "13db9db575bf47db8f23becbd9e01019": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "1576424bae8d4428805f27f725184d09": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "172c9b77254048e492f902e4701ae4f4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_138d4dda99b4450a88a311faea032010", + "IPY_MODEL_4a26297c8a634c68b1024bc178cfa1d3", + "IPY_MODEL_f8ab59fad4d54c11b8ecb28a135f77ee" ], - "metadata": { - "id": "P_YLvPDCVb2z" - } + "layout": "IPY_MODEL_1e3823ff48a64f719da05bb5522477e3" + } }, - { - "cell_type": "code", - "source": [ - "from datasets import load_dataset\n", - "from haystack import Document\n", - "\n", - "dataset = load_dataset(\"bilgeyucel/seven-wonders\", split=\"train\")\n", - "docs = [Document(content=doc[\"content\"], meta=doc[\"meta\"]) for doc in dataset]" + "17d4d36595184416bdbbb2722d324127": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "1bc14d35ee3c4fd1bfe53713c410cb4e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_85c78672211245b8af111b88341192f9", + "placeholder": "​", + "style": "IPY_MODEL_37955fbdf27e4b278e1ed53efb00ae03", + "value": "Batches: 100%" + } + }, + "1c1acd6c555e48a088036c3c4b19d6a8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4de8ed8dfc8a47f694ddcf7fc05470ce", + "placeholder": "​", + "style": "IPY_MODEL_6135c2c5b28c4189b4c5600ae11c7f93", + "value": "model.safetensors: 100%" + } + }, + "1c76d5b3076746c2a2d78b5851f7bfa2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_89d9afef1c344a72ba188d1ba621d1af", + "max": 295, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_00debc7e71fc4d6bb0af6d8a6f25c0f7", + "value": 295 + } + }, + "1ca743af07ab42829fd961acdd641a4a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_90dfee714fa14024aa0235709c8da661", + "placeholder": "​", + "style": "IPY_MODEL_1f711847a2314067a2ffd5fd60dbd959", + "value": "vocab.txt: 100%" + } + }, + "1da668f10ae3483da7de191430c211fb": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "1e2e4c55c2044fb7965649506009efc0": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "1e3823ff48a64f719da05bb5522477e3": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "1f1be98d3b294254a0ac88aafd6ce67c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "1f711847a2314067a2ffd5fd60dbd959": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "20177ae3a9f14622a1fef3196159cb44": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a203aaa4067a4335b9f61bb81b6006e5", + "placeholder": "​", + "style": "IPY_MODEL_17d4d36595184416bdbbb2722d324127", + "value": "special_tokens_map.json: 100%" + } + }, + "215cf162756c4da19ec6f84792a39b69": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "24ffa535eee34c48820d1cdc107cb484": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_7c99612b86bf428d9ce4d7b4440caff4", + "max": 1, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_1da668f10ae3483da7de191430c211fb", + "value": 1 + } + }, + "266bc71b03824c42b0f93f98dd299784": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "268db15fd9a3450caa5c0e5a242121c4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a368ee64770e47bca6980d658fad932a", + "placeholder": "​", + "style": "IPY_MODEL_940070f87ee745139d725399dd649392", + "value": " 5/5 [00:00<00:00,  6.77it/s]" + } + }, + "26e71a1ef9b740e0b6c59a088c17b241": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "277b0f8022654e8e957845cd9ee57493": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "27ab028d35804b8fa1a8023b5e8525e6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "2b0ea9f3052043ff9f239fcf108c2f6d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "2d7cfb3c17a440b0988d5b9a71e9c634": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_fe1f34d442d14f0dbf428c86150f59aa", + "placeholder": "​", + "style": "IPY_MODEL_e956b75c71bb480ba366220c6eea62f6", + "value": " 1.36M/1.36M [00:00<00:00, 1.56MB/s]" + } + }, + "2fc62a3826dc4c7e95145ac24cfdb40d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_f3e3192dcead450d89fd7dd2a57a8a3c", + "placeholder": "​", + "style": "IPY_MODEL_fb70857ca8e44d399ae0525f099ca230", + "value": " 239/239 [00:00<00:00, 15.1kB/s]" + } + }, + "300a3c39844d40588ab543b85f2f29c4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "35f35dadb3c649ebba63a8cf1ea7ab99": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "36fe5bad3e4e434c8128132da61af293": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "37955fbdf27e4b278e1ed53efb00ae03": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "3a5f55591c4d4e6d9f2439898a0780c2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "409eb7e9ed9e4bc1ba6fe9e324a55cdc": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d58ed27dd0134542a43afedca9254f8d", + "placeholder": "​", + "style": "IPY_MODEL_f675a27210c74769a6fdafacb484e2d6", + "value": " 1/1 [00:00<00:00, 27.45it/s]" + } + }, + "425ebd00eaa54edfb531f00e08cc90cf": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "435532e97b19419fb0240cc9c2f2a6dc": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_644ebc3128f24e6daae1856e142c0386", + "placeholder": "​", + "style": "IPY_MODEL_b6354198917840c4bc942b9cd4667f93", + "value": "tokenizer_config.json: 100%" + } + }, + "444170e3f8834d21983f89095aebdae3": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "4603de989e4640efbf7c379e5c7640dc": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "47d0530844184daf9bc90c68ddb4f9fa": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4a26297c8a634c68b1024bc178cfa1d3": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_c498bcf3c5aa4251a3969d26b1b0e770", + "max": 334, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_99858d8e2b67469db0bcb1f87a029c99", + "value": 334 + } + }, + "4a8706a789d5479493bf122020513e88": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4c3f1c744fab4ade95eba8125e116d4d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_fd00db3af05c486b8c8d04a60366be51", + "max": 798293, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_bc7c67380dfc45668f9cd430e8516000", + "value": 798293 + } + }, + "4d3cfdeb1bd24db8824f00468b1635c5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4603de989e4640efbf7c379e5c7640dc", + "max": 619, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_1f1be98d3b294254a0ac88aafd6ce67c", + "value": 619 + } + }, + "4de8ed8dfc8a47f694ddcf7fc05470ce": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4e664743e8e24c40aab9a4b4bf4d4ec6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_1c1acd6c555e48a088036c3c4b19d6a8", + "IPY_MODEL_52a5a268f56a41c3bd39db3a05268192", + "IPY_MODEL_a58e79dda9ae4548b7b774282dac7559" ], - "metadata": { - "id": "esXQClYnVeoM" - }, - "execution_count": null, - "outputs": [] + "layout": "IPY_MODEL_b2c23aa910c146869303f99f29d6ac14" + } }, - { - "cell_type": "markdown", - "source": [ - "### Initalize a Document Embedder\n", - "\n", - "To store your data in the DocumentStore with embeddings, initialize a [SentenceTransformersDocumentEmbedder](https://docs.haystack.deepset.ai/docs/sentencetransformersdocumentembedder) with the model name and call `warm_up()` to download the embedding model.\n", - "\n", - "> If you'd like, you can use a different [Embedder](https://docs.haystack.deepset.ai/docs/embedders) for your documents." + "4f02f9ae1623496f8754406eb24527b1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "5020e9035ba8423ea51109a23a95bfd2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ffb7ed4a6f984068a46330e3bae16027", + "max": 1355881, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_8ce0c21b77dd4bcea15a730eb6f128c0", + "value": 1355881 + } + }, + "517fab9a3db348dc8964fbf921e9f61f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_5b1dd24a09b14c8eaa5c7601ebf3705e", + "IPY_MODEL_8fe73a8d53bb403c9c2eb35efcd536fd", + "IPY_MODEL_dad54c47738d4afea261cc1cee2df8fd" ], - "metadata": { - "id": "Ghm4qrNqVhaA" - } + "layout": "IPY_MODEL_47d0530844184daf9bc90c68ddb4f9fa" + } }, - { - "cell_type": "code", - "source": [ - "from haystack.components.embedders import SentenceTransformersDocumentEmbedder\n", - "\n", - "doc_embedder = SentenceTransformersDocumentEmbedder(model=\"sentence-transformers/all-MiniLM-L6-v2\")\n", - "doc_embedder.warm_up()" + "52a5a268f56a41c3bd39db3a05268192": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_215cf162756c4da19ec6f84792a39b69", + "max": 44696790, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_c2ad3379af7b4a83932c670117b6ac4d", + "value": 44696790 + } + }, + "52e9d9c6c4804b5392d413e2b1f70203": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b204177f854b45768b270e3c69e6a266", + "placeholder": "​", + "style": "IPY_MODEL_277b0f8022654e8e957845cd9ee57493", + "value": " 496M/496M [00:13<00:00, 36.3MB/s]" + } + }, + "53aa58c50eea49e1874ccdbca72af8ec": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "5627acc9660a496d84d65b1f8c280bdb": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "594e4a67550549d7ab6e4ceb70bd3ee8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_c978c043a67a40dd9497143f98b8ac87", + "placeholder": "​", + "style": "IPY_MODEL_956c5d4526b643f7bb5592d4477f82f2", + "value": " 798k/798k [00:00<00:00, 1.15MB/s]" + } + }, + "5b0ee86da7254efebee0292ffcaaa9fb": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "5b1dd24a09b14c8eaa5c7601ebf3705e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_9fbeeebbbbe347f7a9572e5fb234fcfd", + "placeholder": "​", + "style": "IPY_MODEL_300a3c39844d40588ab543b85f2f29c4", + "value": "merges.txt: 100%" + } + }, + "5dc9b503c5094dce8849694e2a6dfaeb": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5e15e24548b044bfb864eb8cc5b712c9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_d8c1fe2c43ec4b8b95092488f138cd39", + "IPY_MODEL_4d3cfdeb1bd24db8824f00468b1635c5", + "IPY_MODEL_f9052e0559544554b01b4afb1206dce8" ], - "metadata": { - "id": "hqGNjp5lVkbv" - }, - "execution_count": null, - "outputs": [] + "layout": "IPY_MODEL_6d73cba2cf3f4fb9b25934700736d41b" + } }, - { - "cell_type": "markdown", - "source": [ - "### Write Documents to the DocumentStore\n", - "\n", - "Run the `doc_embedder` with the Documents. The embedder will create embeddings for each document and save these embeddings in Document object's `embedding` field. Then, you can write the Documents to the DocumentStore with `write_documents()` method." + "5f95340e313746a1b1c7a6f27abb4600": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "6072b48ab7fb4d0fac14902eb4f609bd": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "609d36079b144c16bc6d6bf49d0062f7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "6135c2c5b28c4189b4c5600ae11c7f93": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "63698c2fa2c645fc8df856e1b7c914bb": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "63dcf941fcc94f47bb7d590a5e2f54b8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "644ebc3128f24e6daae1856e142c0386": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "67001f19b6d5442f8f77f7bb0695a8ea": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_ba7e6ff00aca4f0ab5c44a3ccf65bb3a", + "IPY_MODEL_4c3f1c744fab4ade95eba8125e116d4d", + "IPY_MODEL_594e4a67550549d7ab6e4ceb70bd3ee8" ], - "metadata": { - "id": "_wzljJVYWQP4" - } + "layout": "IPY_MODEL_6a3f9587f7ed4f7fa207542f9351f905" + } }, - { - "cell_type": "code", - "source": [ - "docs_with_embeddings = doc_embedder.run(docs)\n", - "document_store.write_documents(docs_with_embeddings[\"documents\"])" + "6a3f9587f7ed4f7fa207542f9351f905": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "6ac3897894094303979a5a675a0a0c4c": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "6b7f104b9356477e81eba214768fc9d9": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "6d73cba2cf3f4fb9b25934700736d41b": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "770d46aa7dca4ad497d3d5c64ce5147e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_13db9db575bf47db8f23becbd9e01019", + "placeholder": "​", + "style": "IPY_MODEL_86d6f5dbdaa24e648a0e1600b7f2d115", + "value": " 112/112 [00:00<00:00, 6.46kB/s]" + } + }, + "7bcc9b32caf84596991cb41bb5931fad": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7c99612b86bf428d9ce4d7b4440caff4": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "807f775eb0a249cf91367f70b0791c60": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "821ebc0e98a3460fa5844658a4f66405": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "85c78672211245b8af111b88341192f9": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "86d6f5dbdaa24e648a0e1600b7f2d115": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "86d7ab02d471454ebb7560adf2af3734": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_c07495fe464e4793af8d7d0ab6cbe68b", + "IPY_MODEL_f55a41788e4a49c9844b4b02274f1c35", + "IPY_MODEL_9f777bed57e5486ab6d833b65155c9ec" ], - "metadata": { - "id": "yxXpVm07WS6D", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 67, - "referenced_widgets": [ - "fe2b5ce2eaed470892fd2de0a3bf09c8", - "1bc14d35ee3c4fd1bfe53713c410cb4e", - "db03553f0ae14d028fd1bea90f8cbc80", - "268db15fd9a3450caa5c0e5a242121c4", - "807f775eb0a249cf91367f70b0791c60", - "85c78672211245b8af111b88341192f9", - "37955fbdf27e4b278e1ed53efb00ae03", - "894091b62a084a538a39e58f93d3ea6e", - "1e2e4c55c2044fb7965649506009efc0", - "a368ee64770e47bca6980d658fad932a", - "940070f87ee745139d725399dd649392" - ] - }, - "outputId": "3862c83e-6f0a-4f2a-fd33-eaf09c274840" - }, - "execution_count": 18, - "outputs": [ - { - "output_type": "display_data", - "data": { - "text/plain": [ - "Batches: 0%| | 0/5 [00:00\n", - "🚅 Components\n", - " - text_router: TransformersTextRouter\n", - " - text_embedder: SentenceTransformersTextEmbedder\n", - " - embedding_retriever: InMemoryEmbeddingRetriever\n", - " - bm25_retriever: InMemoryBM25Retriever\n", - " - document_joiner: DocumentJoiner\n", - "🛤️ Connections\n", - " - text_router.LABEL_0 -> text_embedder.text (str)\n", - " - text_router.LABEL_1 -> bm25_retriever.query (str)\n", - " - text_embedder.embedding -> embedding_retriever.query_embedding (List[float])\n", - " - embedding_retriever.documents -> document_joiner.documents (List[Document])\n", - " - bm25_retriever.documents -> document_joiner.documents (List[Document])" - ] - }, - "metadata": {}, - "execution_count": 20 - } + "9270418511e945f2863b2866b81a2168": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "93e5e993baf346b39205fab6d988da84": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_0c4a42c7897d4da1b3e7b62007196854", + "IPY_MODEL_24ffa535eee34c48820d1cdc107cb484", + "IPY_MODEL_409eb7e9ed9e4bc1ba6fe9e324a55cdc" ], - "source": [ - "from haystack import Pipeline\n", - "\n", - "query_classification_pipeline = Pipeline()\n", - "query_classification_pipeline.add_component(\"text_router\", text_router)\n", - "query_classification_pipeline.add_component(\"text_embedder\", text_embedder)\n", - "query_classification_pipeline.add_component(\"embedding_retriever\", embedding_retriever)\n", - "query_classification_pipeline.add_component(\"bm25_retriever\", bm25_retriever)\n", - "query_classification_pipeline.add_component(\"document_joiner\", document_joiner)\n", - "\n", - "query_classification_pipeline.connect(\"text_router.LABEL_0\", \"text_embedder\")\n", - "query_classification_pipeline.connect(\"text_embedder\", \"embedding_retriever\")\n", - "query_classification_pipeline.connect(\"text_router.LABEL_1\", \"bm25_retriever\")\n", - "query_classification_pipeline.connect(\"bm25_retriever\", \"document_joiner\")\n", - "query_classification_pipeline.connect(\"embedding_retriever\", \"document_joiner\")" - ] + "layout": "IPY_MODEL_35f35dadb3c649ebba63a8cf1ea7ab99" + } }, - { - "cell_type": "markdown", - "metadata": { - "id": "imqRRCGTwQav" - }, - "source": [ - "### 4) Run the Pipeline\n", - "\n", - "Below, you can see how this choice affects the branching structure: the keyword query \"arya stark father\" and the question query \"Who is the father of Arya Stark?\" generate noticeably different results, a distinction that is likely due to the use of different retrievers for keyword vs. question/statement queries." - ] + "93f4cadf8aad48d19d60d4861386926b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d88cb0502ba649d9b01fdbcd3d059b11", + "max": 496254452, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_5b0ee86da7254efebee0292ffcaaa9fb", + "value": 496254452 + } }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": { - "collapsed": true, - "id": "fP6Cpcb-o0HK", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 355, - "referenced_widgets": [ - "93e5e993baf346b39205fab6d988da84", - "0c4a42c7897d4da1b3e7b62007196854", - "24ffa535eee34c48820d1cdc107cb484", - "409eb7e9ed9e4bc1ba6fe9e324a55cdc", - "35f35dadb3c649ebba63a8cf1ea7ab99", - "dcf6276e44cd491c912d1b09502fc91b", - "63dcf941fcc94f47bb7d590a5e2f54b8", - "7c99612b86bf428d9ce4d7b4440caff4", - "1da668f10ae3483da7de191430c211fb", - "d58ed27dd0134542a43afedca9254f8d", - "f675a27210c74769a6fdafacb484e2d6" - ] - }, - "outputId": "a6bfc181-c8f3-4d01-fe90-a4f5d787eae2" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "\n", - "\n", - "==============================\n", - "QUESTION QUERY RESULTS\n", - "==============================\n", - "{'document_joiner': {'documents': [Document(id=4c82325818ccd91af8d68fec37108ce7a93696392f315bd0497ad3a8903d0b45, content: 'The Masonic House of the Temple of the Scottish Rite, Washington, DC, designed by John Russell Pope,...', meta: {'url': 'https://en.wikipedia.org/wiki/Mausoleum_at_Halicarnassus', '_split_id': 18}, score: 8.192663165691801, embedding: vector of size 384), Document(id=4a988f268c10bbb6af9a18063a14460b7e0126c7ed1befb2be17c9cbbc4bb064, content: 'The earliest pharaonic name of seal impressions is that of Khufu, the latest of Pepi II. Worker graf...', meta: {'url': 'https://en.wikipedia.org/wiki/Great_Pyramid_of_Giza', '_split_id': 4}, score: 6.652366985087608, embedding: vector of size 384), Document(id=9c1d7b92058a18bd101c037c87519e9225983c1dbb9386d51412895d5101d096, content: '[68]\n", - "The most notable account of this legend was given by Al-Masudi (896–956) in his Akbar al-zaman,...', meta: {'url': 'https://en.wikipedia.org/wiki/Great_Pyramid_of_Giza', '_split_id': 19}, score: 5.365566894478934, embedding: vector of size 384), Document(id=1ac6d4aedca0f952d87a1cfd4d4bde946d54f103a8f802419671d9ad2e26f3bc, content: '[21] However, the gardens were said to still exist at the time that later writers described them, an...', meta: {'url': 'https://en.wikipedia.org/wiki/Hanging_Gardens_of_Babylon', '_split_id': 5}, score: 5.279086706756248, embedding: vector of size 384), Document(id=975a20d36be68ebf6c7c7a37012dd0deff29797201dddb21e1b3a25c29b168e2, content: 'Modern historians have pointed out that two years would not be enough time to decorate and build suc...', meta: {'url': 'https://en.wikipedia.org/wiki/Mausoleum_at_Halicarnassus', '_split_id': 5}, score: 4.983734342490586, embedding: vector of size 384), Document(id=5d02e37e955d05869cad244b01e9c458c0676a736390b04f347390aa6d8236f7, content: 'The passage is 2 cubits (1.0 m; 3.4 ft) wide and 1.17 m (3.8 ft) high for most of its length, but ne...', meta: {'url': 'https://en.wikipedia.org/wiki/Great_Pyramid_of_Giza', '_split_id': 45}, score: 4.9689671066012755, embedding: vector of size 384), Document(id=06ce9e11253c0c619159c6bd9c8f8e711d7a841025efc696e07b74000c5f750a, content: 'The remains were described briefly by Strabo (64 or 63 BC – c. 24 AD), in his work Geography (Book X...', meta: {'url': 'https://en.wikipedia.org/wiki/Colossus_of_Rhodes', '_split_id': 6}, score: 4.956812213009003, embedding: vector of size 384), Document(id=8898fdd9ba5e7049fcec94ae5e9f172337de969977ccbfb1d5e2b23baf964d15, content: 'Only Josephus names Nebuchadnezzar as the king who built the gardens; although Nebuchadnezzar left m...', meta: {'url': 'https://en.wikipedia.org/wiki/Hanging_Gardens_of_Babylon', '_split_id': 7}, score: 4.950459961177993, embedding: vector of size 384), Document(id=e42388e23323dad64d9cd5dd5e93fa1391d93be3dc161236d789e2eb7de99546, content: 'These were entrusted to a boat builder, Haj Ahmed Yusuf, who worked out how the pieces fit together....', meta: {'url': 'https://en.wikipedia.org/wiki/Great_Pyramid_of_Giza', '_split_id': 60}, score: 4.917705216995133, embedding: vector of size 384), Document(id=417cb72c998c669851f4cfc8b4a2ba75c7ab8fd2e880f53ea10e6c75d8e0a6b2, content: 'Further, he describes a method discovered by Thales of Miletus for ascertaining the pyramid's height...', meta: {'url': 'https://en.wikipedia.org/wiki/Great_Pyramid_of_Giza', '_split_id': 17}, score: 4.9033381526508935, embedding: vector of size 384)]}}\n" - ] - }, - { - "output_type": "stream", - "name": "stderr", - "text": [ - "/usr/local/lib/python3.10/dist-packages/transformers/pipelines/text_classification.py:104: UserWarning: `return_all_scores` is now deprecated, if want a similar functionality use `top_k=None` instead of `return_all_scores=True` or `top_k=1` instead of `return_all_scores=False`.\n", - " warnings.warn(\n" - ] - }, - { - "output_type": "display_data", - "data": { - "text/plain": [ - "Batches: 0%| | 0/1 [00:00\n", - "🚅 Components\n", - " - bm25_retriever_0: InMemoryBM25Retriever\n", - " - bm25_retriever_1: InMemoryBM25Retriever\n", - " - text_router: TransformersTextRouter\n", - " - reader: ExtractiveReader\n", - "🛤️ Connections\n", - " - bm25_retriever_0.documents -> reader.documents (List[Document])\n", - " - text_router.LABEL_0 -> bm25_retriever_0.query (str)\n", - " - text_router.LABEL_1 -> bm25_retriever_1.query (str)" - ] - }, - "metadata": {}, - "execution_count": 22 - } + "c9dc77a6e6c64c328e49a0f2bea925b1": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d2d3a829d57f40a5af01e74bf9919cfa": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_9c2c4061fe7247c59eea7dddb37de56a", + "IPY_MODEL_0e1cffb58c94491ca7246ffb3d9e0657", + "IPY_MODEL_2fc62a3826dc4c7e95145ac24cfdb40d" ], - "source": [ - "from haystack.components.readers import ExtractiveReader\n", - "\n", - "query_classification_pipeline = Pipeline()\n", - "query_classification_pipeline.add_component(\"bm25_retriever_0\", InMemoryBM25Retriever(document_store))\n", - "query_classification_pipeline.add_component(\"bm25_retriever_1\", InMemoryBM25Retriever(document_store))\n", - "query_classification_pipeline.add_component(\"text_router\", TransformersTextRouter(model=\"shahrukhx01/question-vs-statement-classifier\"))\n", - "query_classification_pipeline.add_component(\"reader\", ExtractiveReader())\n", - "\n", - "query_classification_pipeline.connect(\"text_router.LABEL_0\", \"bm25_retriever_0\")\n", - "query_classification_pipeline.connect(\"bm25_retriever_0\", \"reader\")\n", - "query_classification_pipeline.connect(\"text_router.LABEL_1\", \"bm25_retriever_1\")" - ] + "layout": "IPY_MODEL_b1c20311cec84d528f8050f5de69abc3" + } }, - { - "cell_type": "markdown", - "metadata": { - "id": "QU1B6JQEDrol" - }, - "source": [ - "### 2) Run the Pipeline\n", - "And here are the results of this pipeline: with a question query like \"Who is the father of Arya Stark?\", you obtain answers from a Reader, and with a statement query like \"Arya Stark was the daughter of a Lord\", you just obtain documents from a Retriever." - ] + "d58b712e8dda486d8a3def5a2e46c8fa": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": { - "id": "HIjgs5k7C6CN", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 493, - "referenced_widgets": [ - "86d7ab02d471454ebb7560adf2af3734", - "c07495fe464e4793af8d7d0ab6cbe68b", - "f55a41788e4a49c9844b4b02274f1c35", - "9f777bed57e5486ab6d833b65155c9ec", - "96dc7d5082474169a8ed0fe90fda29be", - "bcd896e543794c2fb0df897931cedcf6", - "d58b712e8dda486d8a3def5a2e46c8fa", - "8e0bb920cc15493db671961d76985397", - "6072b48ab7fb4d0fac14902eb4f609bd", - "bc2d103625d547c78a7e14011710c5c3", - "53aa58c50eea49e1874ccdbca72af8ec", - "c8c7816cb48f46ca8b419f8c7efb8f5e", - "b400314ef3644290ac4f4e86ef3bf17b", - "93f4cadf8aad48d19d60d4861386926b", - "52e9d9c6c4804b5392d413e2b1f70203", - "dae3785870ff4d3b872798cc76ce6dcd", - "6ac3897894094303979a5a675a0a0c4c", - "f2cbfde5e1044ed388a94defa478d5a5", - "d88cb0502ba649d9b01fdbcd3d059b11", - "5b0ee86da7254efebee0292ffcaaa9fb", - "b204177f854b45768b270e3c69e6a266", - "277b0f8022654e8e957845cd9ee57493", - "e0f6aea68492445b976f2dde5c87cf16", - "435532e97b19419fb0240cc9c2f2a6dc", - "1c76d5b3076746c2a2d78b5851f7bfa2", - "bcb3c70d00fd43b6ac66ace3153a6d7e", - "5627acc9660a496d84d65b1f8c280bdb", - "644ebc3128f24e6daae1856e142c0386", - "b6354198917840c4bc942b9cd4667f93", - "89d9afef1c344a72ba188d1ba621d1af", - "00debc7e71fc4d6bb0af6d8a6f25c0f7", - "e9ee9ef7c4724e6d8ea20bdd6d6ef080", - "5f95340e313746a1b1c7a6f27abb4600", - "67001f19b6d5442f8f77f7bb0695a8ea", - "ba7e6ff00aca4f0ab5c44a3ccf65bb3a", - "4c3f1c744fab4ade95eba8125e116d4d", - "594e4a67550549d7ab6e4ceb70bd3ee8", - "6a3f9587f7ed4f7fa207542f9351f905", - "4a8706a789d5479493bf122020513e88", - "609d36079b144c16bc6d6bf49d0062f7", - "fd00db3af05c486b8c8d04a60366be51", - "bc7c67380dfc45668f9cd430e8516000", - "c978c043a67a40dd9497143f98b8ac87", - "956c5d4526b643f7bb5592d4477f82f2", - "517fab9a3db348dc8964fbf921e9f61f", - "5b1dd24a09b14c8eaa5c7601ebf3705e", - "8fe73a8d53bb403c9c2eb35efcd536fd", - "dad54c47738d4afea261cc1cee2df8fd", - "47d0530844184daf9bc90c68ddb4f9fa", - "9fbeeebbbbe347f7a9572e5fb234fcfd", - "300a3c39844d40588ab543b85f2f29c4", - "e7291573b13542c7a0d80795a974c4d1", - "df66f8d6b8d6494782143e1776cbca7f", - "c9dc77a6e6c64c328e49a0f2bea925b1", - "b4da651ee89f48c99cbc7b7bbaf7c884", - "9033ee66e6da494d85ae616fc0efba1e", - "9f2a6b2692ab45c4900842b93c3a4742", - "5020e9035ba8423ea51109a23a95bfd2", - "2d7cfb3c17a440b0988d5b9a71e9c634", - "266bc71b03824c42b0f93f98dd299784", - "63698c2fa2c645fc8df856e1b7c914bb", - "9270418511e945f2863b2866b81a2168", - "ffb7ed4a6f984068a46330e3bae16027", - "8ce0c21b77dd4bcea15a730eb6f128c0", - "fe1f34d442d14f0dbf428c86150f59aa", - "e956b75c71bb480ba366220c6eea62f6", - "d2d3a829d57f40a5af01e74bf9919cfa", - "9c2c4061fe7247c59eea7dddb37de56a", - "0e1cffb58c94491ca7246ffb3d9e0657", - "2fc62a3826dc4c7e95145ac24cfdb40d", - "b1c20311cec84d528f8050f5de69abc3", - "5dc9b503c5094dce8849694e2a6dfaeb", - "27ab028d35804b8fa1a8023b5e8525e6", - "0328a80806e84a60821b74200f6581f5", - "bf3bec07aa4946fc8fa02359098c0cc3", - "f3e3192dcead450d89fd7dd2a57a8a3c", - "fb70857ca8e44d399ae0525f099ca230" - ] - }, - "outputId": "f3f94168-433e-4955-89b2-ae33d245b084" - }, - "outputs": [ - { - "output_type": "display_data", - "data": { - "text/plain": [ - "config.json: 0%| | 0.00/729 [00:00