From ac5583f2f69373fda60ae9abe9e19569790b4f62 Mon Sep 17 00:00:00 2001 From: Giovanni Tommasini Date: Fri, 3 Jan 2025 14:35:57 +0100 Subject: [PATCH 1/2] Use qdrant:latest in docker-compose-local.yaml --- docker-compose-local.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose-local.yaml b/docker-compose-local.yaml index 5b4d597..b5f4e0d 100644 --- a/docker-compose-local.yaml +++ b/docker-compose-local.yaml @@ -13,7 +13,7 @@ services: qdrant: container_name: qdrant_demo_qdrant - image: qdrant/qdrant:v1.7.3 + image: qdrant/qdrant:latest ports: - "6333:6333" - "6334:6334" From 85effd6dd67e8ec2b87879282197780f4e1737b2 Mon Sep 17 00:00:00 2001 From: Giovanni Tommasini Date: Fri, 3 Jan 2025 14:36:14 +0100 Subject: [PATCH 2/2] Implement log in qdrant_demo/init_collection_startups.py and fix optimizers_config for new version of qdrant client --- qdrant_demo/init_collection_startups.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/qdrant_demo/init_collection_startups.py b/qdrant_demo/init_collection_startups.py index 962b126..94fae22 100644 --- a/qdrant_demo/init_collection_startups.py +++ b/qdrant_demo/init_collection_startups.py @@ -1,5 +1,7 @@ import json import os.path +import logging +from datetime import datetime from qdrant_client import QdrantClient, models from tqdm import tqdm @@ -8,42 +10,49 @@ def upload_embeddings(): + print(f"{datetime.now().strftime('%Y-%m-%d %H:%M:%S,%f')[:-3]} - INFO - Initializing Qdrant client...") client = QdrantClient( url=QDRANT_URL, api_key=QDRANT_API_KEY, prefer_grpc=True, ) + print(f"{datetime.now().strftime('%Y-%m-%d %H:%M:%S,%f')[:-3]} - INFO - Setting embedding model: {EMBEDDINGS_MODEL}") client.set_model(EMBEDDINGS_MODEL) payload_path = os.path.join(DATA_DIR, 'startups_demo.json') + print(f"{datetime.now().strftime('%Y-%m-%d %H:%M:%S,%f')[:-3]} - INFO - Loading data from file: {payload_path}") payload = [] documents = [] with open(payload_path) as fd: + print(f"{datetime.now().strftime('%Y-%m-%d %H:%M:%S,%f')[:-3]} - INFO - Processing data...") for line in fd: obj = json.loads(line) - # Rename fields to unified schema documents.append(obj.pop('description')) obj["logo_url"] = obj.pop("images") obj["homepage_url"] = obj.pop("link") payload.append(obj) + + print(f"{datetime.now().strftime('%Y-%m-%d %H:%M:%S,%f')[:-3]} - INFO - Data processed: {len(documents)} documents") + print(f"{datetime.now().strftime('%Y-%m-%d %H:%M:%S,%f')[:-3]} - INFO - Recreating collection: {COLLECTION_NAME}") client.recreate_collection( collection_name=COLLECTION_NAME, vectors_config=client.get_fastembed_vector_params(on_disk=True), - # Quantization is optional, but it can significantly reduce the memory usage quantization_config=models.ScalarQuantization( scalar=models.ScalarQuantizationConfig( type=models.ScalarType.INT8, quantile=0.99, always_ram=True ) + ), + optimizers_config=models.OptimizersConfigDiff( + max_optimization_threads=2 ) ) - # Create a payload index for text field. - # This index enables text search by the TEXT_FIELD_NAME field. + print(f"{datetime.now().strftime('%Y-%m-%d %H:%M:%S,%f')[:-3]} - INFO - Creating payload index for field: {TEXT_FIELD_NAME}") client.create_payload_index( collection_name=COLLECTION_NAME, field_name=TEXT_FIELD_NAME, @@ -56,6 +65,7 @@ def upload_embeddings(): ) ) + print(f"{datetime.now().strftime('%Y-%m-%d %H:%M:%S,%f')[:-3]} - INFO - Uploading documents to collection...") client.add( collection_name=COLLECTION_NAME, documents=documents, @@ -63,6 +73,8 @@ def upload_embeddings(): ids=tqdm(range(len(payload))), parallel=0, ) + + print(f"{datetime.now().strftime('%Y-%m-%d %H:%M:%S,%f')[:-3]} - INFO - Upload completed successfully!") if __name__ == '__main__':