From b8f79b7743401f8efc4bcdec83a4b8824020d4b6 Mon Sep 17 00:00:00 2001 From: irumvanselme Date: Tue, 10 Mar 2026 11:39:52 +0200 Subject: [PATCH 1/4] feat(server): add swagger UI parameter to collapse documentation sections --- backend/app/server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/app/server.py b/backend/app/server.py index 576912b60..1e71f82c9 100644 --- a/backend/app/server.py +++ b/backend/app/server.py @@ -314,6 +314,7 @@ async def lifespan(_app: FastAPI): version=get_application_config().version_info.to_version_string(), description=f"The {_global_product_name} API is used to interact with the {_global_product_name} conversation agent.", redirect_slashes=False, + swagger_ui_parameters={"docExpansion": "none"}, servers=[ { "url": backend_url or "/", From 1366934e36683e25d33f112eefdffd04e2e01832 Mon Sep 17 00:00:00 2001 From: irumvanselme Date: Mon, 9 Mar 2026 15:50:30 +0200 Subject: [PATCH 2/4] feat(esco_search): optimize skill aggregation pipeline for improved performance --- .../app/vector_search/esco_search_service.py | 61 +++++++++---------- 1 file changed, 29 insertions(+), 32 deletions(-) diff --git a/backend/app/vector_search/esco_search_service.py b/backend/app/vector_search/esco_search_service.py index 41b5db6a5..6f99d6a85 100644 --- a/backend/app/vector_search/esco_search_service.py +++ b/backend/app/vector_search/esco_search_service.py @@ -388,8 +388,14 @@ async def _find_skills_of_occupation(self, occupation: OccupationEntity): if cached_result is not None: return cached_result - skills = await self.relations_collection.aggregate([ + pipeline = [ {"$match": {"modelId": self._model_id, "requiringOccupationId": ObjectId(occupation.id)}}, + {"$project": { + "requiredSkillId": 1, + "modelId": 1, + "relationType": 1, + "signallingValueLabel": 1, + }}, { "$lookup": { "from": self.embedding_config.skill_collection_name, @@ -397,42 +403,33 @@ async def _find_skills_of_occupation(self, occupation: OccupationEntity): "foreignField": "skillId", "as": "skills", "pipeline": [ - {"$match": {"modelId": self._model_id}} + {"$match": {"modelId": self._model_id}}, + # Limit 1 of the skills (Note: each skill has 3 corresponding documents, pick the first. + {"$limit": 1}, + # Exclude large/irrelevant fields from the result to reduce network transfer and improve latency + {"$project": { "embedding": 0, "embedded_field": 0, "embedded_text": 0, }} ] } }, - {"$unwind": "$skills"}, - {"$group": {"_id": "$skills.skillId", - "modelId": {"$first": "$modelId"}, - "skillId": {"$first": "$skills.skillId"}, - "UUID": {"$first": "$skills.UUID"}, - "preferredLabel": {"$first": "$skills.preferredLabel"}, - "description": {"$first": "$skills.description"}, - "scopeNote": {"$first": "$skills.scopeNote"}, - "originUUID": {"$first": "$skills.originUUID"}, - "UUIDHistory": {"$first": "$skills.UUIDHistory"}, - "altLabels": {"$first": "$skills.altLabels"}, - "skillType": {"$first": "$skills.skillType"}, - "relationType": {"$first": "$relationType"}, - "signallingValueLabel": {"$first": "$signallingValueLabel"}, - } - } - ]).to_list(length=None) + {"$unwind": "$skills"} + ] + + skills_relationships = await self.relations_collection.aggregate(pipeline).to_list(length=None) result = [AssociatedSkillEntity( - id=str(skill.get("skillId", "")), - modelId=str(skill.get("modelId", "")), - UUID=skill.get("UUID", ""), - preferredLabel=skill.get("preferredLabel", ""), - description=skill.get("description", ""), - scopeNote=skill.get("scopeNote", ""), - altLabels=skill.get("altLabels", []), - skillType=skill.get("skillType", ""), - originUUID=skill.get("originUUID", ""), - UUIDHistory=skill.get("UUIDHistory", []), - relationType=skill.get("relationType", ""), - signallingValueLabel=skill.get("signallingValueLabel", ""), + id=str(skill_relationship.get("skills").get("skillId", "")), + modelId=str(skill_relationship.get("modelId", "")), + UUID=skill_relationship.get("skills").get("UUID", ""), + preferredLabel=skill_relationship.get("skills").get("preferredLabel", ""), + description=skill_relationship.get("skills").get("description", ""), + scopeNote=skill_relationship.get("skills").get("scopeNote", ""), + altLabels=skill_relationship.get("skills").get("altLabels", []), + skillType=skill_relationship.get("skills").get("skillType", ""), + originUUID=skill_relationship.get("skills").get("originUUID", ""), + UUIDHistory=skill_relationship.get("skills").get("UUIDHistory", []), + relationType=skill_relationship.get("relationType", ""), + signallingValueLabel=skill_relationship.get("signallingValueLabel", ""), score=0.0 - ) for skill in skills] + ) for skill_relationship in skills_relationships] # Cache the result await _skills_of_occupation_cache.set(occupation.id, result) From bf424f618ab94c9ace7911480fc409be67af0162 Mon Sep 17 00:00:00 2001 From: irumvanselme Date: Mon, 9 Mar 2026 16:02:39 +0200 Subject: [PATCH 3/4] feat(index): create compound index on skillId and modelId for optimized MongoDB lookups to fix the issues of index missing on some forks --- backend/scripts/embeddings/_common.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/backend/scripts/embeddings/_common.py b/backend/scripts/embeddings/_common.py index d0c5926af..30c53360a 100644 --- a/backend/scripts/embeddings/_common.py +++ b/backend/scripts/embeddings/_common.py @@ -285,6 +285,29 @@ async def create_skills_indexes(*, hot_run: bool, id_field_name="skillId", logger=logger, ) + + # Create an index on (skillId, modelId). + # This index optimizes MongoDB $lookup operations where documents + # are searched by `skillId` within a specific `modelId`. + # + # The default compound index order (modelId, skillId) is not optimal + # for this query pattern because MongoDB can miss the index when the + # leading field is not used first. In some cases this causes the query + # planner to skip the index and perform a less efficient scan. + # + # Therefore, we explicitly create the index with `skillId` as the + # leading field to match the lookup access pattern. + await _upsert_index( + hot_run=hot_run, + collection=collection, + keys={ + "skillId": 1, + "modelId": 1, + }, + name="skill_id_model_id_index", + logger=logger, + ) + await _create_vector_search_index(hot_run=hot_run, collection=collection, num_of_dimensions=num_of_dimensions, From 2943fc2fe2ccbc79eb71195c324d121364ec09ae Mon Sep 17 00:00:00 2001 From: irumvanselme Date: Tue, 10 Mar 2026 11:35:08 +0200 Subject: [PATCH 4/4] refactor(generate_taxonomy_embeddings): do not validate the model details if the script is only generating indexes --- .../generate_taxonomy_embeddings.py | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/backend/scripts/embeddings/generate_taxonomy_embeddings.py b/backend/scripts/embeddings/generate_taxonomy_embeddings.py index eb3e49eb0..815e1279d 100755 --- a/backend/scripts/embeddings/generate_taxonomy_embeddings.py +++ b/backend/scripts/embeddings/generate_taxonomy_embeddings.py @@ -404,21 +404,21 @@ async def main(opts: Options): logger.info("Starting the main function") logger.info("Using options: " + opts.__str__()) - if opts.delete_existing: - # Delete existing relations and model info collections - for collection in CompassEmbeddingsCollections: - await delete_existing(hot_run=opts.hot_run, - collection_name=collection.value, - model_id=SCRIPT_SETTINGS.tabiya_model_id) + # [1/5] Copy the model info and validate for existing state, if it is compatible with the new state. + if opts.generate_embeddings: + if opts.delete_existing: + # Delete existing relations and model info collections + for collection in CompassEmbeddingsCollections: + await delete_existing(hot_run=opts.hot_run, + collection_name=collection.value, + model_id=SCRIPT_SETTINGS.tabiya_model_id) - embeddings_service = await get_embeddings_service(service_name=SCRIPT_SETTINGS.embeddings_service_name, - model_name=SCRIPT_SETTINGS.embeddings_model_name) + embeddings_service = await get_embeddings_service(service_name=SCRIPT_SETTINGS.embeddings_service_name, + model_name=SCRIPT_SETTINGS.embeddings_model_name) - # [1/5] Copy the model info and validate for existing state, if it is compatible with the new state. - await _copy_model_info(hot_run=opts.hot_run, embeddings_service=embeddings_service) + await _copy_model_info(hot_run=opts.hot_run, embeddings_service=embeddings_service) - # run the three tasks in parallel - if opts.generate_embeddings: + # run the three tasks in parallel await asyncio.gather( # [2/5] Copy the relations collection copy_relations_collection(hot_run=opts.hot_run), @@ -438,8 +438,8 @@ async def main(opts: Options): num_of_dimensions=opts.num_of_dimensions, logger=logger) + logger.info("Script execution completed") -logger.info("Script execution completed") if __name__ == "__main__": try: