From 0c5bb389fc5172caf8e2b3616d21cf6960b416e5 Mon Sep 17 00:00:00 2001 From: "zhenjie.ye" Date: Wed, 29 Nov 2023 14:31:07 +0800 Subject: [PATCH 1/2] add AzureOpenAI Embedding Support --- .../packages/gizmo-agent/gizmo_agent/ingest.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/backend/packages/gizmo-agent/gizmo_agent/ingest.py b/backend/packages/gizmo-agent/gizmo_agent/ingest.py index e1e41c01a..1d6e06c35 100644 --- a/backend/packages/gizmo-agent/gizmo_agent/ingest.py +++ b/backend/packages/gizmo-agent/gizmo_agent/ingest.py @@ -2,6 +2,7 @@ from agent_executor.upload import IngestRunnable from langchain.embeddings import OpenAIEmbeddings +from langchain.embeddings.azure_openai import AzureOpenAIEmbeddings from langchain.schema.runnable import ConfigurableField from langchain.text_splitter import RecursiveCharacterTextSplitter from langchain.vectorstores.redis import Redis @@ -9,16 +10,28 @@ index_schema = { "tag": [{"name": "namespace"}], } + +if os.environ("OPENAI_API_TYPE") == "azure": + embeddings = OpenAIEmbeddings( + deployment=os.environ["AZURE_OPENAI_DEPLOYMENT_NAME_EB"], + model=os.environ["AZURE_OPENAI_DEPLOYMENT_NAME_EB"], + openai_api_base=os.environ["AZURE_OPENAI_API_BASE"], + openai_api_type="azure", + ) +else: + embeddings = OpenAIEmbeddings() + vstore = Redis( redis_url=os.environ["REDIS_URL"], index_name="opengpts", - embedding=OpenAIEmbeddings(), + embedding=embeddings, index_schema=index_schema, ) ingest_runnable = IngestRunnable( - text_splitter=RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200), + text_splitter=RecursiveCharacterTextSplitter( + chunk_size=1000, chunk_overlap=200), vectorstore=vstore, ).configurable_fields( assistant_id=ConfigurableField( From aedde4a3d063281f08974b596afcac4ef86c43d9 Mon Sep 17 00:00:00 2001 From: "zhenjie.ye" Date: Wed, 29 Nov 2023 15:15:23 +0800 Subject: [PATCH 2/2] add AzureOpenAI Embedding Model Support --- .env | 20 +++++++++++++++++++ .../gizmo-agent/gizmo_agent/ingest.py | 3 +-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/.env b/.env index cae8618bd..a5cabaa7d 100644 --- a/.env +++ b/.env @@ -1,2 +1,22 @@ +OPENAI_API_TYPE=placeholder +# if OPENAI_API_TYPE is "azure", then the OPENAI_API_KEY is the Azure API key OPENAI_API_KEY=placeholder + +AZURE_OPENAI_API_BASE=placeholder +AZURE_OPENAI_API_VERSION=placeholder +AZURE_OPENAI_API_KEY=placeholder +AZURE_OPENAI_DEPLOYMENT_NAME=placeholder +AZURE_OPENAI_DEPLOYMENT_NAME_EB="your_azure_embedding_deployment_name_here" + +REDIS_URL=placeholder + ANTHROPIC_API_KEY=placeholder +YDC_API_KEY=your_secret_here +LANGCHAIN_TRACING_V2="true" +LANGCHAIN_PROJECT=langserve-launch-example +LANGCHAIN_API_KEY=your_secret_key_here +TAVILY_API_KEY=your_secret_here +FIREWORKS_API_KEY=your_secret_here +AWS_ACCESS_KEY_ID=your_secret_here +AWS_SECRET_ACCESS_KEY=your_secret_here +KAY_API_KEY=your_secret_here \ No newline at end of file diff --git a/backend/packages/gizmo-agent/gizmo_agent/ingest.py b/backend/packages/gizmo-agent/gizmo_agent/ingest.py index 1d6e06c35..294a1b0ab 100644 --- a/backend/packages/gizmo-agent/gizmo_agent/ingest.py +++ b/backend/packages/gizmo-agent/gizmo_agent/ingest.py @@ -2,7 +2,6 @@ from agent_executor.upload import IngestRunnable from langchain.embeddings import OpenAIEmbeddings -from langchain.embeddings.azure_openai import AzureOpenAIEmbeddings from langchain.schema.runnable import ConfigurableField from langchain.text_splitter import RecursiveCharacterTextSplitter from langchain.vectorstores.redis import Redis @@ -11,7 +10,7 @@ "tag": [{"name": "namespace"}], } -if os.environ("OPENAI_API_TYPE") == "azure": +if os.environ["OPENAI_API_TYPE"] == "azure": embeddings = OpenAIEmbeddings( deployment=os.environ["AZURE_OPENAI_DEPLOYMENT_NAME_EB"], model=os.environ["AZURE_OPENAI_DEPLOYMENT_NAME_EB"],