feat: Added French version of Entity Linker

.gitignore

@@ -238,4 +238,7 @@ fabric.properties
 .idea/caches/build_file_checksums.ser
 
 # Mac Desktop Services Store
-.DS_Store
+.DS_Store
+
+# poetry
+poetry.lock

inference/evaluator.py

@@ -8,21 +8,20 @@
 import logging
 from tqdm import trange
 from sentence_transformers.util import cos_sim, dot_score
-from sentence_transformers.evaluation import SentenceEvaluator
+from sentence_transformers.sentence_transformer.evaluation import SentenceEvaluator
 import os
 import numpy as np
 from typing import List, Dict, Set, Tuple
 import pandas as pd
 from linker import EntityLinker
 
 
-
 class InformationRetrievalEvaluator(SentenceEvaluator):
     """
     This class evaluates an Information Retrieval (IR) setting.
 
     Given a set of queries and a large corpus set. It will retrieve for each query the top-k most similar document. It measures
-    Mean Reciprocal Rank (MRR), Recall@k, and Normalized Discounted Cumulative Gain (NDCG)
+    Mean Reciprocal Rank (MRR), Recall@k, Normalized Discounted Cumulative Gain (NDCG), and RPrecision@k.
     """
 
     def __init__(
@@ -36,6 +35,7 @@ def __init__(
         accuracy_at_k: List[int] = [1, 4, 16, 32],
         precision_recall_at_k: List[int] = [1, 4, 16, 32],
         map_at_k: List[int] = [1, 4, 5, 16, 32],
+        r_precision_at_k: List[int] = [1, 4, 16, 32], # <-- Added RP@K Initialization
         show_progress_bar: bool = False,
         batch_size: int = 32,
         name: str = "",
@@ -56,15 +56,13 @@ def __init__(
         self.accuracy_at_k = accuracy_at_k
         self.precision_recall_at_k = precision_recall_at_k
         self.map_at_k = map_at_k
+        self.r_precision_at_k = r_precision_at_k # <-- Set internal state
 
         self.show_progress_bar = show_progress_bar
         self.batch_size = batch_size
         self.name = name
         self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
-
-
-
     def compute_metrics(self, queries_result_list: List[object]):
         # Init score computation values
         num_hits_at_k = {k: 0 for k in self.accuracy_at_k}
@@ -73,6 +71,7 @@ def compute_metrics(self, queries_result_list: List[object]):
         MRR = {k: 0 for k in self.mrr_at_k}
         ndcg = {k: [] for k in self.ndcg_at_k}
         AveP_at_k = {k: [] for k in self.map_at_k}
+        R_Precision_at_k = {k: [] for k in self.r_precision_at_k} # <-- Init RP@K dict
 
         # Compute scores on results
         for query_itr in range(len(queries_result_list)):
@@ -82,6 +81,7 @@ def compute_metrics(self, queries_result_list: List[object]):
             #top_hits = sorted(queries_result_list[query_itr], key=lambda x: x["score"], reverse=True)
             top_hits = queries_result_list[query_itr]
             query_relevant_docs = self.relevant_docs[query_id]
+            Rn = len(query_relevant_docs) # Total relevant docs for this query
 
             # Accuracy@k - We count the result correct, if at least one relevant doc is across the top-k documents
             for k_val in self.accuracy_at_k:
@@ -98,7 +98,19 @@ def compute_metrics(self, queries_result_list: List[object]):
                         num_correct += 1
 
                 precisions_at_k[k_val].append(num_correct / k_val)
-                recall_at_k[k_val].append(num_correct / len(query_relevant_docs))
+                recall_at_k[k_val].append(num_correct / Rn)
+
+            # RPrecision@k <-- NEW METRIC CALCULATION
+            for k_val in self.r_precision_at_k:
+                num_correct = 0
+                for hit in top_hits[0:k_val]:
+                    if hit["corpus_id"] in query_relevant_docs:
+                        num_correct += 1
+
+                # RP@K Formula: sum(Rel(n,k)) / min(K, R_n)
+                denominator = min(k_val, Rn)
+                r_precision_score = num_correct / denominator if denominator > 0 else 0.0
+                R_Precision_at_k[k_val].append(r_precision_score)
 
             # MRR@k
             for k_val in self.mrr_at_k:
@@ -112,7 +124,7 @@ def compute_metrics(self, queries_result_list: List[object]):
                 predicted_relevance = [
                     1 if top_hit["corpus_id"] in query_relevant_docs else 0 for top_hit in top_hits[0:k_val]
                 ]
-                true_relevances = [1] * len(query_relevant_docs)
+                true_relevances = [1] * Rn
 
                 ndcg_value = self.compute_dcg_at_k(predicted_relevance, k_val) / self.compute_dcg_at_k(
                     true_relevances, k_val
@@ -129,7 +141,7 @@ def compute_metrics(self, queries_result_list: List[object]):
                         num_correct += 1
                         sum_precisions += num_correct / (rank + 1)
 
-                avg_precision = sum_precisions / min(k_val, len(query_relevant_docs))
+                avg_precision = sum_precisions / min(k_val, Rn)
                 AveP_at_k[k_val].append(avg_precision)
 
         # Compute averages
@@ -150,6 +162,10 @@ def compute_metrics(self, queries_result_list: List[object]):
 
         for k in AveP_at_k:
             AveP_at_k[k] = np.mean(AveP_at_k[k])
+
+        # Average the new RPrecision metric across all N queries
+        for k in R_Precision_at_k:
+            R_Precision_at_k[k] = np.mean(R_Precision_at_k[k])
 
         return {
             "accuracy@k": num_hits_at_k,
@@ -158,6 +174,7 @@ def compute_metrics(self, queries_result_list: List[object]):
             "ndcg@k": ndcg,
             "mrr@k": MRR,
             "map@k": AveP_at_k,
+            "r_precision@k": R_Precision_at_k, # <-- Output new metric
         }
 
     @staticmethod
@@ -213,10 +230,10 @@ def _load_dataset(self) -> dict:
     if self.entity_type=="Skill":
 
       #Load necessary files in pandas dataframes
-      dftech_val = pd.read_csv(os.path.join(self.dir, 'eval', 'tech_validation_annotations.csv'))
-      dftech_test = pd.read_csv(os.path.join(self.dir, 'eval', 'tech_test_annotations.csv'))
-      dfhouse_val = pd.read_csv(os.path.join(self.dir, 'eval', 'house_validation_annotations.csv'))
-      dfhouse_test = pd.read_csv(os.path.join(self.dir, 'eval', 'house_test_annotations.csv'))
+      dftech_val = pd.read_csv(os.path.join(self.dir, 'eval', 'tech_validation_annotations_no_unk.csv'))
+      dftech_test = pd.read_csv(os.path.join(self.dir, 'eval', 'tech_test_annotations_no_unk.csv'))
+      dfhouse_val = pd.read_csv(os.path.join(self.dir, 'eval', 'house_validation_annotations_no_unk.csv'))
+      dfhouse_test = pd.read_csv(os.path.join(self.dir, 'eval', 'house_test_annotations_no_unk.csv'))
       #Prepeocess
       df = pd.concat([dftech_val, dftech_test, dfhouse_val, dfhouse_test], ignore_index=True)
       df = df.drop_duplicates(ignore_index=True)

inference/linker.py

@@ -55,6 +55,9 @@ class EntityLinker:
 	output_format : str, default='occupation'
 		Specifies the format of the output for occupations, either `occupation`, `preffered_label`, `esco_code`, `uuid` or `all` to get all the columns. 
 		The `uuid` is also available for the skills.
+
+	lang : str, default='en'
+		Specifies the language of the reference sets to load. Currently supports 'en' for English and 'fr' for French.
 
 	Calling Parameters
 	----------
@@ -73,7 +76,8 @@ def __init__(
 			evaluation_mode: bool = False,
 			k: int = 32,
 			from_cache: bool = True,
-			output_format: str = 'occupation'
+			output_format: str = 'occupation',
+			lang: str = 'en'
 	):
 		# Initialize the model paths and settings
 		self.entity_model = entity_model
@@ -85,7 +89,7 @@ def __init__(
 		self.from_cache = from_cache
 		self.output_format = output_format
 		self.path_to_files = os.path.abspath(os.path.join(os.path.dirname(__file__), 'files'))
-
+		self.lang = lang
 		# Set the device to GPU if available, otherwise CPU
 		self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 
@@ -102,15 +106,14 @@ def __init__(
 		self.tokenizer = AutoTokenizer.from_pretrained(entity_model, token=os.getenv('HF_TOKEN'))
 
 		# Load reference sets for occupations, skills, and qualifications
-		self.df_occ = pd.read_csv(os.path.join(self.path_to_files, 'occupations_augmented.csv'))
-		self.df_skill = pd.read_csv(os.path.join(self.path_to_files, 'skills.csv'))
-		self.df_qual = pd.read_csv(os.path.join(self.path_to_files, 'qualifications.csv'))
+		self.df_occ, self.df_skill, self.df_qual = self._load_dfs(lang=self.lang)
 
 		# Load precomputed embeddings for the reference sets
+		# TODO: Implement vector database support for scalability
 		self.occupation_emb, self.skill_emb, self.qualification_emb = self._load_tensors()
 
 
-	def __call__(self, text: str, linking: bool = True) -> List[dict]:
+	def __call__(self, text: str, linking: bool = True, el: bool = True) -> List[dict]:
 		"""
 		Perform job-related entity recognition and optionally link entities to a taxonomy.
 
@@ -122,6 +125,9 @@ def __call__(self, text: str, linking: bool = True) -> List[dict]:
 		linking : bool, default=True
 			Specifies whether the model should perform the entity linking to the taxonomy. 
 			If `False`, it might only extract entities without linking them to a predefined taxonomy.
+
+		el : bool, default=True
+			Specifies whether the model should perform entity linking (el=True) or sentence linking (el=False).
 
 		Returns
 		-------
@@ -148,18 +154,24 @@ def __call__(self, text: str, linking: bool = True) -> List[dict]:
 		#     text = UtilFunctions.translate(text)
 
 		# Sentence tokenize with nltk to handle lengthy inputs.
-		text_list = sent_tokenize(text)
-		output = []
-
-		# Process each sentence in the text
-		for item in text_list:
-			# Run the model on each sentence and extend the output list with the results
-			output.extend(self._run_model(item, linking)) if self._run_model(item, linking) else None
-
+		if el:
+			text_list = sent_tokenize(text)
+			output = []
+
+			# Process each sentence in the text
+			for item in text_list:
+				# Run the model on each sentence and extend the output list with the results
+				entities = self._run_el_model(item, linking)
+				if entities:
+					output.extend(entities)
+		else:
+			output = self._run_sl_model(text)
+		#TODO : Add a post-processing step to aggregate entities across sentences. This should be optional.
+		#TODO : Apply sentence linking to link entities across sentences.
 		return output
 
 
-	def _run_model(self, sentence: str, link: bool) -> List[dict]:
+	def _run_el_model(self, sentence: str, link: bool) -> List[dict]:
 		"""
 		Perform entity extraction and optionally link entities to the ESCO taxonomie.
 
@@ -200,6 +212,44 @@ def _run_model(self, sentence: str, link: bool) -> List[dict]:
 						entry['retrieved'] = self._top_k(emb, entry['type'])
 
 		return formatted_entities
+
+	def _run_sl_model(self, sentence: str) -> List[dict]:
+		"""
+		Perform entity linking for a given sentence without entity extraction.
+		Parameters
+		----------
+		sentence : str
+			A sentence from which to link entities to the ESCO taxonomy.
+		Returns
+		-------
+		List[dict]
+			A dictionary with the sentence and the top-k most similar entities from the reference sets.
+			The dictionary contains the following
+			keys:
+			- `sentence`: The input sentence.
+			- `occupations`: A dictionary with keys:
+			- `text`: A list of the top-k most similar occupation names or ESCO codes from the reference set.
+			- `scores`: (Optional) If `evaluation_mode` is `True`, a list of cosine similarity scores for the retrieved occupations.
+		"""
+		# Initialize the output dictionary
+		formatted_output = {'sentence': sentence, 
+					  'occupations': {'text':[], 'scores':[]}, 
+					  'skills': {'text':[], 'scores':[]}, 
+					  'qualifications': {'text':[], 'scores':[]}}
+		# Encode the sentence into embeddings
+		emb = self.similarity_model.encode(sentence)
+		emb = torch.from_numpy(emb).to(self.device)
+		# Retrieve top-k suggestions for each entity type
+		for entity_type in ['Occupation', 'Skill', 'Qualification']:
+			# Retrieve the top-k suggestions based on the sentence embedding
+			if self.evaluation_mode:
+				retrieved, scores = self._top_k(emb, entity_type)
+				formatted_output[entity_type.lower() + 's']['text'] += retrieved
+				formatted_output[entity_type.lower() + 's']['scores'] += scores
+			else:
+				retrieved = self._top_k(emb, entity_type)
+				formatted_output[entity_type.lower() + 's']['text'] += retrieved
+		return formatted_output
 
 
 	def _ner_pipeline(self, text: str) -> List[dict]:
@@ -334,7 +384,7 @@ def _load_tensors(self) -> Tuple[List[torch.Tensor]]:
 		"""
 
 		# Determine the path for storing or loading the embeddings
-		path = os.path.join(self.path_to_files, self.similarity_model_type)
+		path = os.path.join(self.path_to_files, self.similarity_model_type.split('/')[-1])
 
 		if self.from_cache:
 			# Load cached embeddings from precomputed files
@@ -481,4 +531,31 @@ def remove_special_tokens_and_tags(input_ids:List[int], bio_tags:List[str], toke
 						filtered_ids.append(id_)
 						filtered_tags.append(tag)
 
-		return filtered_ids, filtered_tags
+		return filtered_ids, filtered_tags
+
+	def _load_dfs(self, lang: str = 'en') -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
+		"""
+		Load the reference DataFrames for occupations, skills, and qualifications based on the specified language.
+
+		Parameters
+		----------
+		lang : str, default='en'
+			The language code for loading the appropriate DataFrames. 
+			Currently supports 'en' for English and 'fr' for French.
+
+		Returns
+		-------
+		Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]
+			A tuple containing three DataFrames: occupations, skills, and qualifications.
+		"""
+		path = self.path_to_files
+		if lang == 'fr':
+			df_occ = pd.read_csv(os.path.join(path, 'occupations_fr.csv'))
+			df_skill = pd.read_csv(os.path.join(path, 'skills_fr.csv'))
+			df_qual = pd.read_csv(os.path.join(path, 'qualifications_fr.csv'))
+		else:
+			df_occ = pd.read_csv(os.path.join(path, 'occupations_augmented.csv'))
+			df_skill = pd.read_csv(os.path.join(path, 'skills.csv'))
+			df_qual = pd.read_csv(os.path.join(path, 'qualifications.csv'))
+
+		return df_occ, df_skill, df_qual