From e78ad8a42a16af7c43f8e1c1449be8741c9a5e4d Mon Sep 17 00:00:00 2001 From: dasgoutam Date: Wed, 2 Apr 2025 14:18:08 +0200 Subject: [PATCH 01/28] add crossencoder model --- integreat_chat/core/settings.py | 3 ++ .../management/commands/opensearch_setup.py | 5 +-- integreat_chat/search/services/opensearch.py | 34 ++++++++++++++++--- 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/integreat_chat/core/settings.py b/integreat_chat/core/settings.py index 9ff1f941..b0e1f52e 100644 --- a/integreat_chat/core/settings.py +++ b/integreat_chat/core/settings.py @@ -101,6 +101,9 @@ SEARCH_SCORE_THRESHOLD = 0.1 SEARCH_MAX_PAGES = 10 SEARCH_EMBEDDING_MODEL_NAME = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2" +OPENSEARCH_CROSSENCODER_MODEL_NAME = ( + "huggingface/cross-encoders/ms-marco-MiniLM-L-6-v2" +) OPENSEARCH_EMBEDDING_MODEL_NAME = ( "huggingface/sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2" ) diff --git a/integreat_chat/search/management/commands/opensearch_setup.py b/integreat_chat/search/management/commands/opensearch_setup.py index 602156dc..4f4dbb7f 100644 --- a/integreat_chat/search/management/commands/opensearch_setup.py +++ b/integreat_chat/search/management/commands/opensearch_setup.py @@ -14,12 +14,13 @@ class Command(BaseCommand): def handle(self, *args, **options): oss = OpenSearchSetup(password=settings.OPENSEARCH_PASSWORD) - group_id, model_id = oss.setup() + group_id, model_id, model_id_ce = oss.setup() self.stdout.write( self.style.SUCCESS( # pylint: disable=no-member f'Successfully set up OpenSearch. Change the following settings ' f'in the OPENSEARCH section of your config:\n' f'MODEL_GROUP_ID = {group_id}\n' - f'MODEL_ID = {model_id}\n' + f'MODEL_ID_EMBEDDING_MODEL = {model_id}\n' + f'MODEL_ID_CROSSENCODER_MODEL = {model_id_ce}\n' ) ) diff --git a/integreat_chat/search/services/opensearch.py b/integreat_chat/search/services/opensearch.py index 8505c620..4966e110 100644 --- a/integreat_chat/search/services/opensearch.py +++ b/integreat_chat/search/services/opensearch.py @@ -325,13 +325,15 @@ def setup(self) -> str: group_id = self.create_model_group() if not group_id: raise ValueError("Unexpected OpenSearch response while creating model group") - model_id = self.register_embedding_model(group_id) - if not model_id: + model_id_embedding = self.register_embedding_model(group_id) + model_id_crossencoder = self.register_crossencoder_model(group_id) + if not model_id_embedding or not model_id_crossencoder: raise ValueError("Unexpected OpenSearch response while registering model") - self.deploy_model(model_id) - self.create_ingestion_pipeline(model_id) + self.deploy_model(model_id_embedding) + self.deploy_model(model_id_crossencoder) + self.create_ingestion_pipeline(model_id_embedding) self.create_search_pipeline() - return group_id, model_id + return group_id, model_id_embedding, model_id_crossencoder def delete_model_group(self): """ @@ -376,6 +378,28 @@ def create_model_group(self): if "model_group_id" in response: return response["model_group_id"] return False + + def register_crossencoder_model(self, model_group_id: str) -> str: + """ + Register crossencoder model + """ + payload = { + "name": settings.OPENSEARCH_CROSSENCODER_MODEL_NAME, + "version": "1.0.2", + "model_group_id": model_group_id, + "model_format": "TORCH_SCRIPT" + } + register_response = self.request( + "/_plugins/_ml/models/_register", payload, "POST" + ) + if "task_id" in register_response: + for n in range(0, 10): + time.sleep(5) + if "model_id" in (task_response := self.request( + f"/_plugins/_ml/tasks/{register_response['task_id']}", {}, "GET" + )): + return task_response["model_id"] + return False def register_embedding_model(self, model_group_id: str) -> str: """ From 23d434c596947f1f3ca3c71e9498e7e5b3486c8c Mon Sep 17 00:00:00 2001 From: dasgoutam Date: Sun, 6 Apr 2025 19:07:14 +0200 Subject: [PATCH 02/28] test search pipeline --- integreat_chat/search/services/opensearch.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/integreat_chat/search/services/opensearch.py b/integreat_chat/search/services/opensearch.py index 4966e110..54bfe5f4 100644 --- a/integreat_chat/search/services/opensearch.py +++ b/integreat_chat/search/services/opensearch.py @@ -485,6 +485,20 @@ def create_search_pipeline(self): } } } + ], + "response_processors": [ + { + "rerank": { + "ml_opensearch": { + "model_id": "CFEa95UBuqG3mdqtTmwA" + }, + "context": { + "document_fields": [ + "passage_text" + ] + } + } + } ] } self.request(f"/_search/pipeline/{self.search_pipeline_name}", payload, "PUT") From c1bde092cc21e126b0bee6eac27c67f7a37555f5 Mon Sep 17 00:00:00 2001 From: dasgoutam Date: Tue, 8 Apr 2025 15:26:01 +0200 Subject: [PATCH 03/28] change pipeline params --- integreat_chat/search/services/opensearch.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/integreat_chat/search/services/opensearch.py b/integreat_chat/search/services/opensearch.py index 54bfe5f4..2aca8d98 100644 --- a/integreat_chat/search/services/opensearch.py +++ b/integreat_chat/search/services/opensearch.py @@ -144,6 +144,13 @@ def search(self, region_slug: str, language_slug: str, message: str) -> dict: } ] } + }, + "ext": { + "rerank": { + "query_context": { + "query_text": message + } + } } } return self.request( @@ -494,7 +501,7 @@ def create_search_pipeline(self): }, "context": { "document_fields": [ - "passage_text" + "chunk_text" ] } } From 6b844a31b1ac3559d110ceb998dd9a4b24c139d1 Mon Sep 17 00:00:00 2001 From: dasgoutam Date: Wed, 9 Apr 2025 15:41:53 +0200 Subject: [PATCH 04/28] add model id of reranker --- integreat_chat/core/settings.py | 2 ++ integreat_chat/search/services/opensearch.py | 3 +++ 2 files changed, 5 insertions(+) diff --git a/integreat_chat/core/settings.py b/integreat_chat/core/settings.py index b0e1f52e..27027a46 100644 --- a/integreat_chat/core/settings.py +++ b/integreat_chat/core/settings.py @@ -118,7 +118,9 @@ SEARCH_DENSE_WEIGHT = 0.7 SEARCH_FALLBACK_LANGUAGE = "en" SEARCH_OPENSEARCH_MODEL_ID = config["OPENSEARCH"]["MODEL_ID"] +SEARCH_OPENSEARCH_MODEL_ID_RERANKER = config["OPENSEARCH"]["MODEL_ID_RERANKER"] SEARCH_OPENSEARCH_MODEL_GROUP_ID = config["OPENSEARCH"]["MODEL_GROUP_ID"] + OPENSEARCH_USER = ( config["OPENSEARCH"]["USER"] if "USER" in config["OPENSEARCH"] diff --git a/integreat_chat/search/services/opensearch.py b/integreat_chat/search/services/opensearch.py index 2aca8d98..27141872 100644 --- a/integreat_chat/search/services/opensearch.py +++ b/integreat_chat/search/services/opensearch.py @@ -36,6 +36,7 @@ def __init__( self.user = user self.password = password self.model_id = settings.SEARCH_OPENSEARCH_MODEL_ID + self.model_id_reranker = settings.SEARCH_OPENSEARCH_MODEL_ID_RERANKER self.model_group_id = settings.SEARCH_OPENSEARCH_MODEL_GROUP_ID def request(self, path: str, payload: dict, method: str = "GET") -> dict: @@ -348,6 +349,8 @@ def delete_model_group(self): """ self.request(f"/_plugins/_ml/models/{self.model_id}/_undeploy", {}, "POST") self.request(f"/_plugins/_ml/models/{self.model_id}", {}, "DELETE") + self.request(f"/_plugins/_ml/models/{self.model_id_reranker}/_undeploy", {}, "POST") + self.request(f"/_plugins/_ml/models/{self.model_id_reranker}", {}, "DELETE") self.request(f"/_plugins/_ml/model_groups/{self.model_group_id}", {}, "DELETE") def prepare_index(self, region_slug: str = "", language_slug: str = ""): From 206b48e5ceecce76a3250a91647722c82be0e5fb Mon Sep 17 00:00:00 2001 From: dasgoutam Date: Wed, 9 Apr 2025 15:58:15 +0200 Subject: [PATCH 05/28] add var for reranker model in search pipeline --- integreat_chat/search/services/opensearch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integreat_chat/search/services/opensearch.py b/integreat_chat/search/services/opensearch.py index 27141872..18f83249 100644 --- a/integreat_chat/search/services/opensearch.py +++ b/integreat_chat/search/services/opensearch.py @@ -500,7 +500,7 @@ def create_search_pipeline(self): { "rerank": { "ml_opensearch": { - "model_id": "CFEa95UBuqG3mdqtTmwA" + "model_id": self.model_id_reranker }, "context": { "document_fields": [ From 29741bd1c7e1b940254355be4ee8565806950544 Mon Sep 17 00:00:00 2001 From: dasgoutam Date: Wed, 16 Apr 2025 15:52:35 +0200 Subject: [PATCH 06/28] try by field reranking --- integreat_chat/search/services/opensearch.py | 21 +++++--------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/integreat_chat/search/services/opensearch.py b/integreat_chat/search/services/opensearch.py index 18f83249..58c8d4e1 100644 --- a/integreat_chat/search/services/opensearch.py +++ b/integreat_chat/search/services/opensearch.py @@ -146,13 +146,6 @@ def search(self, region_slug: str, language_slug: str, message: str) -> dict: ] } }, - "ext": { - "rerank": { - "query_context": { - "query_text": message - } - } - } } return self.request( f"/{region_slug}_{language_slug}/_search?" @@ -498,15 +491,11 @@ def create_search_pipeline(self): ], "response_processors": [ { - "rerank": { - "ml_opensearch": { - "model_id": self.model_id_reranker - }, - "context": { - "document_fields": [ - "chunk_text" - ] - } + "rerank": { + "by_field": { + "target_field": "chunk_text", + "keep_previous_score" : true + } } } ] From c0219a238fac4f943234510f334561f4fbb0379f Mon Sep 17 00:00:00 2001 From: dasgoutam Date: Wed, 16 Apr 2025 15:59:46 +0200 Subject: [PATCH 07/28] correct typo --- integreat_chat/search/services/opensearch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integreat_chat/search/services/opensearch.py b/integreat_chat/search/services/opensearch.py index 58c8d4e1..f3bf84ed 100644 --- a/integreat_chat/search/services/opensearch.py +++ b/integreat_chat/search/services/opensearch.py @@ -494,7 +494,7 @@ def create_search_pipeline(self): "rerank": { "by_field": { "target_field": "chunk_text", - "keep_previous_score" : true + "keep_previous_score" : True } } } From 47f5ed65b63a4cdd73e90140d826cf2b4f1ec0c7 Mon Sep 17 00:00:00 2001 From: dasgoutam Date: Wed, 30 Apr 2025 11:09:12 +0200 Subject: [PATCH 08/28] revert to cross encoder model --- integreat_chat/search/services/opensearch.py | 27 ++++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/integreat_chat/search/services/opensearch.py b/integreat_chat/search/services/opensearch.py index f3bf84ed..90fd2588 100644 --- a/integreat_chat/search/services/opensearch.py +++ b/integreat_chat/search/services/opensearch.py @@ -146,6 +146,13 @@ def search(self, region_slug: str, language_slug: str, message: str) -> dict: ] } }, + "ext": { + "rerank": { + "query_context": { + "query_text": message + } + } + } } return self.request( f"/{region_slug}_{language_slug}/_search?" @@ -489,16 +496,19 @@ def create_search_pipeline(self): } } ], - "response_processors": [ - { - "rerank": { - "by_field": { - "target_field": "chunk_text", - "keep_previous_score" : True - } + "response_processors": [ + { + "rerank": { + "ml_opensearch": { + "model_id": self.model_id_reranker + }, + "context": { + "document_fields": [ + "chunk_text" + ] } } - ] + }] } self.request(f"/_search/pipeline/{self.search_pipeline_name}", payload, "PUT") @@ -533,6 +543,7 @@ def create_index(self, index_slug: str): "settings": { "index.knn": True, "default_pipeline": self.ingest_pipeline_name, + "index.search.default_pipeline": self.search_pipeline_name, }, "mappings": { "properties": { From fac8a90fc7148915351d84d06909132a8d1cbe39 Mon Sep 17 00:00:00 2001 From: dasgoutam Date: Wed, 4 Jun 2025 13:20:44 +0200 Subject: [PATCH 09/28] increase number of docs --- integreat_chat/core/settings.py | 4 ++-- integreat_chat/search/services/opensearch.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/integreat_chat/core/settings.py b/integreat_chat/core/settings.py index 27027a46..ae8f3a46 100644 --- a/integreat_chat/core/settings.py +++ b/integreat_chat/core/settings.py @@ -97,8 +97,8 @@ RAG_FALLBACK_LANGUAGE = "en" # SEARCH_MAX_DOCUMENTS - number of documents retrieved from the VDB -SEARCH_MAX_DOCUMENTS = 15 -SEARCH_SCORE_THRESHOLD = 0.1 +SEARCH_MAX_DOCUMENTS = 20 +SEARCH_SCORE_THRESHOLD = 0 SEARCH_MAX_PAGES = 10 SEARCH_EMBEDDING_MODEL_NAME = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2" OPENSEARCH_CROSSENCODER_MODEL_NAME = ( diff --git a/integreat_chat/search/services/opensearch.py b/integreat_chat/search/services/opensearch.py index 90fd2588..2258233f 100644 --- a/integreat_chat/search/services/opensearch.py +++ b/integreat_chat/search/services/opensearch.py @@ -130,7 +130,7 @@ def search(self, region_slug: str, language_slug: str, message: str) -> dict: "title_embedding": { "query_text": message, "model_id": self.model_id, - "k": 5 + "k": 20 } } }, @@ -139,7 +139,7 @@ def search(self, region_slug: str, language_slug: str, message: str) -> dict: "chunk_embedding": { "query_text": message, "model_id": self.model_id, - "k": 5 + "k": 20 } } } From 45d4abfe44fb392de971cacb8b1bb45107139b92 Mon Sep 17 00:00:00 2001 From: dasgoutam Date: Wed, 4 Jun 2025 13:34:35 +0200 Subject: [PATCH 10/28] test commit --- integreat_chat/search/services/opensearch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integreat_chat/search/services/opensearch.py b/integreat_chat/search/services/opensearch.py index 2258233f..942a37f7 100644 --- a/integreat_chat/search/services/opensearch.py +++ b/integreat_chat/search/services/opensearch.py @@ -130,7 +130,7 @@ def search(self, region_slug: str, language_slug: str, message: str) -> dict: "title_embedding": { "query_text": message, "model_id": self.model_id, - "k": 20 + "k": 25 } } }, @@ -139,7 +139,7 @@ def search(self, region_slug: str, language_slug: str, message: str) -> dict: "chunk_embedding": { "query_text": message, "model_id": self.model_id, - "k": 20 + "k": 25 } } } From 046324fc1eea22ffa35aca12bccd4b7aa6ac3fe9 Mon Sep 17 00:00:00 2001 From: dasgoutam Date: Wed, 4 Jun 2025 14:31:40 +0200 Subject: [PATCH 11/28] try no score threshold --- integreat_chat/core/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integreat_chat/core/settings.py b/integreat_chat/core/settings.py index ae8f3a46..10225143 100644 --- a/integreat_chat/core/settings.py +++ b/integreat_chat/core/settings.py @@ -98,7 +98,7 @@ # SEARCH_MAX_DOCUMENTS - number of documents retrieved from the VDB SEARCH_MAX_DOCUMENTS = 20 -SEARCH_SCORE_THRESHOLD = 0 +SEARCH_SCORE_THRESHOLD = -100 SEARCH_MAX_PAGES = 10 SEARCH_EMBEDDING_MODEL_NAME = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2" OPENSEARCH_CROSSENCODER_MODEL_NAME = ( From f25dde6f3a74aedf9c48d7c9fd4c27c131a79530 Mon Sep 17 00:00:00 2001 From: dasgoutam Date: Thu, 5 Jun 2025 12:02:14 +0200 Subject: [PATCH 12/28] change crossencoder model --- integreat_chat/core/settings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integreat_chat/core/settings.py b/integreat_chat/core/settings.py index 10225143..3065b6d6 100644 --- a/integreat_chat/core/settings.py +++ b/integreat_chat/core/settings.py @@ -102,7 +102,8 @@ SEARCH_MAX_PAGES = 10 SEARCH_EMBEDDING_MODEL_NAME = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2" OPENSEARCH_CROSSENCODER_MODEL_NAME = ( - "huggingface/cross-encoders/ms-marco-MiniLM-L-6-v2" + #"huggingface/cross-encoders/ms-marco-MiniLM-L-6-v2" + "jinaai/jina-reranker-v2-base-multilingual" ) OPENSEARCH_EMBEDDING_MODEL_NAME = ( "huggingface/sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2" From 82d39afff27f143f9c05219921a2f35e6880034f Mon Sep 17 00:00:00 2001 From: dasgoutam Date: Thu, 5 Jun 2025 12:38:36 +0200 Subject: [PATCH 13/28] add huggingface to reranker model name --- integreat_chat/core/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integreat_chat/core/settings.py b/integreat_chat/core/settings.py index 3065b6d6..dbdafd12 100644 --- a/integreat_chat/core/settings.py +++ b/integreat_chat/core/settings.py @@ -103,7 +103,7 @@ SEARCH_EMBEDDING_MODEL_NAME = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2" OPENSEARCH_CROSSENCODER_MODEL_NAME = ( #"huggingface/cross-encoders/ms-marco-MiniLM-L-6-v2" - "jinaai/jina-reranker-v2-base-multilingual" + "huggingface/jinaai/jina-reranker-v2-base-multilingual" ) OPENSEARCH_EMBEDDING_MODEL_NAME = ( "huggingface/sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2" From 1816b2b30739f1c821ff1f9b6d2f74c6bd89ed68 Mon Sep 17 00:00:00 2001 From: dasgoutam Date: Thu, 5 Jun 2025 12:43:28 +0200 Subject: [PATCH 14/28] change error messages --- integreat_chat/search/services/opensearch.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/integreat_chat/search/services/opensearch.py b/integreat_chat/search/services/opensearch.py index 942a37f7..ba687e7d 100644 --- a/integreat_chat/search/services/opensearch.py +++ b/integreat_chat/search/services/opensearch.py @@ -335,8 +335,10 @@ def setup(self) -> str: raise ValueError("Unexpected OpenSearch response while creating model group") model_id_embedding = self.register_embedding_model(group_id) model_id_crossencoder = self.register_crossencoder_model(group_id) - if not model_id_embedding or not model_id_crossencoder: - raise ValueError("Unexpected OpenSearch response while registering model") + if not model_id_embedding: + raise ValueError("Unexpected OpenSearch response while registering embedding model") + elif not model_id_crossencoder: + raise ValueError("Unexpected OpenSearch response while registering crossencoder model") self.deploy_model(model_id_embedding) self.deploy_model(model_id_crossencoder) self.create_ingestion_pipeline(model_id_embedding) From 93975fa10cebc11864673bda0c0a5548abdf4261 Mon Sep 17 00:00:00 2001 From: dasgoutam Date: Thu, 5 Jun 2025 15:19:33 +0200 Subject: [PATCH 15/28] change payload model group --- integreat_chat/search/services/opensearch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integreat_chat/search/services/opensearch.py b/integreat_chat/search/services/opensearch.py index ba687e7d..4e391b16 100644 --- a/integreat_chat/search/services/opensearch.py +++ b/integreat_chat/search/services/opensearch.py @@ -383,8 +383,8 @@ def create_model_group(self): Create model group """ payload = { - "name": "integreat-chat-2025-01-31", - "description": "Integreat Chat embedding models" + "name": "integreat-chat-2025-06-05", + "description": "Integreat Chat model group" } response = self.request("/_plugins/_ml/model_groups/_register", payload, "POST") if "model_group_id" in response: From 14167a50bba1e49a66c015e2390b5e35863b802c Mon Sep 17 00:00:00 2001 From: dasgoutam Date: Thu, 5 Jun 2025 20:44:10 +0200 Subject: [PATCH 16/28] add logging statement --- integreat_chat/search/services/opensearch.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/integreat_chat/search/services/opensearch.py b/integreat_chat/search/services/opensearch.py index 4e391b16..89d1bdb8 100644 --- a/integreat_chat/search/services/opensearch.py +++ b/integreat_chat/search/services/opensearch.py @@ -2,6 +2,7 @@ Setup and use of OpenSearch """ +import logging import hashlib import time from datetime import timedelta, datetime @@ -11,6 +12,8 @@ from langchain_text_splitters import HTMLHeaderTextSplitter from integreat_chat.core.utils.integreat_cms import get_all_pages, get_parent_page_titles +LOGGER = logging.getLogger("django") + class OpenSearch: """ Class for searching and updating documents in OpenSearch @@ -387,6 +390,7 @@ def create_model_group(self): "description": "Integreat Chat model group" } response = self.request("/_plugins/_ml/model_groups/_register", payload, "POST") + logging.info(f"Model group response: {response}") if "model_group_id" in response: return response["model_group_id"] return False From 11d585a3311c71a88667c7d9a20a25aff3e5dfc6 Mon Sep 17 00:00:00 2001 From: dasgoutam Date: Thu, 5 Jun 2025 20:46:04 +0200 Subject: [PATCH 17/28] change logging to print --- integreat_chat/search/services/opensearch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integreat_chat/search/services/opensearch.py b/integreat_chat/search/services/opensearch.py index 89d1bdb8..4fd0b3be 100644 --- a/integreat_chat/search/services/opensearch.py +++ b/integreat_chat/search/services/opensearch.py @@ -390,7 +390,7 @@ def create_model_group(self): "description": "Integreat Chat model group" } response = self.request("/_plugins/_ml/model_groups/_register", payload, "POST") - logging.info(f"Model group response: {response}") + print(f"Model group response: {response}") if "model_group_id" in response: return response["model_group_id"] return False From b25fd107a1c372b80f1dffe7fff2007d388028c9 Mon Sep 17 00:00:00 2001 From: dasgoutam Date: Thu, 5 Jun 2025 20:50:17 +0200 Subject: [PATCH 18/28] revert back to logger --- integreat_chat/search/services/opensearch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integreat_chat/search/services/opensearch.py b/integreat_chat/search/services/opensearch.py index 4fd0b3be..7aa1a924 100644 --- a/integreat_chat/search/services/opensearch.py +++ b/integreat_chat/search/services/opensearch.py @@ -390,7 +390,7 @@ def create_model_group(self): "description": "Integreat Chat model group" } response = self.request("/_plugins/_ml/model_groups/_register", payload, "POST") - print(f"Model group response: {response}") + LOGGER.debug(f"Model group response: {response}") if "model_group_id" in response: return response["model_group_id"] return False From 62d296ea09c95233a584323aac7bc314b24805f9 Mon Sep 17 00:00:00 2001 From: dasgoutam Date: Fri, 6 Jun 2025 10:58:44 +0200 Subject: [PATCH 19/28] add logging for model registration --- integreat_chat/search/services/opensearch.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/integreat_chat/search/services/opensearch.py b/integreat_chat/search/services/opensearch.py index 7aa1a924..23d8f3a7 100644 --- a/integreat_chat/search/services/opensearch.py +++ b/integreat_chat/search/services/opensearch.py @@ -408,6 +408,7 @@ def register_crossencoder_model(self, model_group_id: str) -> str: register_response = self.request( "/_plugins/_ml/models/_register", payload, "POST" ) + LOGGER.debug(f"Embedding model response: {register_response}") if "task_id" in register_response: for n in range(0, 10): time.sleep(5) @@ -430,6 +431,7 @@ def register_embedding_model(self, model_group_id: str) -> str: register_response = self.request( "/_plugins/_ml/models/_register", payload, "POST" ) + LOGGER.debug(f"Cross Encoder model response: {register_response}") if "task_id" in register_response: for n in range(0, 10): # pylint: disable=W0612 time.sleep(5) From dcdc7cfa12f18d5f7e5e0b61f1b78ee5959af625 Mon Sep 17 00:00:00 2001 From: dasgoutam Date: Fri, 6 Jun 2025 11:36:30 +0200 Subject: [PATCH 20/28] revert to opensearch pretrained model --- integreat_chat/core/settings.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/integreat_chat/core/settings.py b/integreat_chat/core/settings.py index dbdafd12..10225143 100644 --- a/integreat_chat/core/settings.py +++ b/integreat_chat/core/settings.py @@ -102,8 +102,7 @@ SEARCH_MAX_PAGES = 10 SEARCH_EMBEDDING_MODEL_NAME = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2" OPENSEARCH_CROSSENCODER_MODEL_NAME = ( - #"huggingface/cross-encoders/ms-marco-MiniLM-L-6-v2" - "huggingface/jinaai/jina-reranker-v2-base-multilingual" + "huggingface/cross-encoders/ms-marco-MiniLM-L-6-v2" ) OPENSEARCH_EMBEDDING_MODEL_NAME = ( "huggingface/sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2" From dbcb85b7b637aea7c2046bb1c9f63db3b2b43be9 Mon Sep 17 00:00:00 2001 From: dasgoutam Date: Fri, 6 Jun 2025 11:54:40 +0200 Subject: [PATCH 21/28] convert crossencoder logits using sigmoid --- integreat_chat/search/services/opensearch.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/integreat_chat/search/services/opensearch.py b/integreat_chat/search/services/opensearch.py index 23d8f3a7..2022f6ce 100644 --- a/integreat_chat/search/services/opensearch.py +++ b/integreat_chat/search/services/opensearch.py @@ -6,6 +6,7 @@ import hashlib import time from datetime import timedelta, datetime +import math import requests from django.conf import settings @@ -79,6 +80,7 @@ def reduce_search_result( """ result = [] found_urls = [] + sigmoid = lambda x: 1 / (1 + math.exp(-x)) if "hits" not in response: raise ValueError("Missing hits in result") for document in response["hits"]["hits"]: @@ -91,7 +93,7 @@ def reduce_search_result( "url": document["_source"]["url"], "title": document["_source"]["title"], "parent_titles": document["_source"]["parent_titles"], - "score": document["_score"], + "score": str(sigmoid(int(document["_score"]))), "chunk_text": document["_source"]["chunk_text"], }) found_urls.append(document["_source"]["url"]) From 3db4861595f6efd12ca1c0c7be7d4221764e753a Mon Sep 17 00:00:00 2001 From: dasgoutam Date: Fri, 6 Jun 2025 11:57:54 +0200 Subject: [PATCH 22/28] remove type conversion from sigmoid --- integreat_chat/search/services/opensearch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integreat_chat/search/services/opensearch.py b/integreat_chat/search/services/opensearch.py index 2022f6ce..19a12686 100644 --- a/integreat_chat/search/services/opensearch.py +++ b/integreat_chat/search/services/opensearch.py @@ -93,7 +93,7 @@ def reduce_search_result( "url": document["_source"]["url"], "title": document["_source"]["title"], "parent_titles": document["_source"]["parent_titles"], - "score": str(sigmoid(int(document["_score"]))), + "score": sigmoid(document["_score"]), "chunk_text": document["_source"]["chunk_text"], }) found_urls.append(document["_source"]["url"]) From 78a2ce208e5e00446f99c03bf3c8fe4160c4024f Mon Sep 17 00:00:00 2001 From: dasgoutam Date: Tue, 10 Jun 2025 15:18:59 +0200 Subject: [PATCH 23/28] change search score threshold back to 0.1 --- integreat_chat/core/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integreat_chat/core/settings.py b/integreat_chat/core/settings.py index 10225143..abed2c39 100644 --- a/integreat_chat/core/settings.py +++ b/integreat_chat/core/settings.py @@ -98,7 +98,7 @@ # SEARCH_MAX_DOCUMENTS - number of documents retrieved from the VDB SEARCH_MAX_DOCUMENTS = 20 -SEARCH_SCORE_THRESHOLD = -100 +SEARCH_SCORE_THRESHOLD = 0.1 SEARCH_MAX_PAGES = 10 SEARCH_EMBEDDING_MODEL_NAME = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2" OPENSEARCH_CROSSENCODER_MODEL_NAME = ( From f46004bb31c8ebfb7ecb4fc204976ca189279772 Mon Sep 17 00:00:00 2001 From: dasgoutam Date: Tue, 10 Jun 2025 15:39:09 +0200 Subject: [PATCH 24/28] change score threshold to 0.05 --- integreat_chat/core/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integreat_chat/core/settings.py b/integreat_chat/core/settings.py index abed2c39..ffed2d3f 100644 --- a/integreat_chat/core/settings.py +++ b/integreat_chat/core/settings.py @@ -98,7 +98,7 @@ # SEARCH_MAX_DOCUMENTS - number of documents retrieved from the VDB SEARCH_MAX_DOCUMENTS = 20 -SEARCH_SCORE_THRESHOLD = 0.1 +SEARCH_SCORE_THRESHOLD = 0.05 SEARCH_MAX_PAGES = 10 SEARCH_EMBEDDING_MODEL_NAME = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2" OPENSEARCH_CROSSENCODER_MODEL_NAME = ( From e4843c4acba0c82de74366b2850232544000347a Mon Sep 17 00:00:00 2001 From: dasgoutam Date: Tue, 10 Jun 2025 15:42:53 +0200 Subject: [PATCH 25/28] no threshold --- integreat_chat/core/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integreat_chat/core/settings.py b/integreat_chat/core/settings.py index ffed2d3f..ae8f3a46 100644 --- a/integreat_chat/core/settings.py +++ b/integreat_chat/core/settings.py @@ -98,7 +98,7 @@ # SEARCH_MAX_DOCUMENTS - number of documents retrieved from the VDB SEARCH_MAX_DOCUMENTS = 20 -SEARCH_SCORE_THRESHOLD = 0.05 +SEARCH_SCORE_THRESHOLD = 0 SEARCH_MAX_PAGES = 10 SEARCH_EMBEDDING_MODEL_NAME = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2" OPENSEARCH_CROSSENCODER_MODEL_NAME = ( From 3278a86508d76c553e07683a2bc8da2b874791a3 Mon Sep 17 00:00:00 2001 From: dasgoutam Date: Tue, 10 Jun 2025 15:51:19 +0200 Subject: [PATCH 26/28] change parameters --- integreat_chat/core/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integreat_chat/core/settings.py b/integreat_chat/core/settings.py index ae8f3a46..24f8db64 100644 --- a/integreat_chat/core/settings.py +++ b/integreat_chat/core/settings.py @@ -97,8 +97,8 @@ RAG_FALLBACK_LANGUAGE = "en" # SEARCH_MAX_DOCUMENTS - number of documents retrieved from the VDB -SEARCH_MAX_DOCUMENTS = 20 -SEARCH_SCORE_THRESHOLD = 0 +SEARCH_MAX_DOCUMENTS = 30 +SEARCH_SCORE_THRESHOLD = -50 SEARCH_MAX_PAGES = 10 SEARCH_EMBEDDING_MODEL_NAME = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2" OPENSEARCH_CROSSENCODER_MODEL_NAME = ( From e0474fd2f74d775c98b2d3ea70c02f37831f8507 Mon Sep 17 00:00:00 2001 From: dasgoutam Date: Wed, 11 Jun 2025 12:44:52 +0200 Subject: [PATCH 27/28] change parameters --- integreat_chat/core/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integreat_chat/core/settings.py b/integreat_chat/core/settings.py index 24f8db64..4943d9f2 100644 --- a/integreat_chat/core/settings.py +++ b/integreat_chat/core/settings.py @@ -98,7 +98,7 @@ # SEARCH_MAX_DOCUMENTS - number of documents retrieved from the VDB SEARCH_MAX_DOCUMENTS = 30 -SEARCH_SCORE_THRESHOLD = -50 +SEARCH_SCORE_THRESHOLD = 0.1 SEARCH_MAX_PAGES = 10 SEARCH_EMBEDDING_MODEL_NAME = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2" OPENSEARCH_CROSSENCODER_MODEL_NAME = ( From d3957ac5d9aa0ed52f056a5d21967072e3a684c3 Mon Sep 17 00:00:00 2001 From: dasgoutam Date: Wed, 11 Jun 2025 12:58:11 +0200 Subject: [PATCH 28/28] change k to 30 --- integreat_chat/search/services/opensearch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integreat_chat/search/services/opensearch.py b/integreat_chat/search/services/opensearch.py index 19a12686..3ccd3b33 100644 --- a/integreat_chat/search/services/opensearch.py +++ b/integreat_chat/search/services/opensearch.py @@ -135,7 +135,7 @@ def search(self, region_slug: str, language_slug: str, message: str) -> dict: "title_embedding": { "query_text": message, "model_id": self.model_id, - "k": 25 + "k": 30 } } }, @@ -144,7 +144,7 @@ def search(self, region_slug: str, language_slug: str, message: str) -> dict: "chunk_embedding": { "query_text": message, "model_id": self.model_id, - "k": 25 + "k": 30 } } }