feat: normalize taxonomy IDs before inference

src/common/utils/taxonomy-normalizer.util.spec.ts

@@ -0,0 +1,45 @@
+import {
+  normalizeTaxonomyArray,
+  normalizeTaxonomyValue,
+} from './taxonomy-normalizer.util';
+
+describe('taxonomy-normalizer.util', () => {
+  describe('normalizeTaxonomyValue', () => {
+    it('strips taxonomy prefixes and normalizes whitespace', () => {
+      expect(normalizeTaxonomyValue('en:nigeria')).toBe('nigeria');
+      expect(normalizeTaxonomyValue('fr:ab-agriculture-biologique')).toBe(
+        'ab-agriculture-biologique',
+      );
+      expect(normalizeTaxonomyValue('  en:united_kingdom  ')).toBe(
+        'united kingdom',
+      );
+    });
+
+    it('returns empty string for invalid inputs', () => {
+      expect(normalizeTaxonomyValue('')).toBe('');
+      expect(normalizeTaxonomyValue('   ')).toBe('');
+      expect(normalizeTaxonomyValue(undefined as any)).toBe('');
+      expect(normalizeTaxonomyValue(null as any)).toBe('');
+    });
+  });
+
+  describe('normalizeTaxonomyArray', () => {
+    it('normalizes, deduplicates and preserves order', () => {
+      expect(
+        normalizeTaxonomyArray([
+          'en:nigeria',
+          'en:english',
+          'EN:NIGERIA',
+          '  en:english ',
+          '',
+          null as any,
+        ]),
+      ).toEqual(['nigeria', 'english']);
+    });
+
+    it('returns empty array for undefined/null', () => {
+      expect(normalizeTaxonomyArray()).toEqual([]);
+      expect(normalizeTaxonomyArray(null)).toEqual([]);
+    });
+  });
+});

src/common/utils/taxonomy-normalizer.util.ts

@@ -0,0 +1,36 @@
+export function normalizeTaxonomyValue(value: string): string {
+  if (typeof value !== 'string') return '';
+
+  let normalized = value.trim();
+  if (!normalized) return '';
+
+  normalized = normalized.replace(/^[a-z]{2,3}:/i, '');
+
+  normalized = normalized
+    .toLowerCase()
+    .replace(/_/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+
+  return normalized;
+}
+
+export function normalizeTaxonomyArray(values?: string[] | null): string[] {
+  if (!Array.isArray(values) || values.length === 0) return [];
+
+  const seen = new Set<string>();
+  const result: string[] = [];
+
+  for (const value of values) {
+    if (typeof value !== 'string') continue;
+
+    const normalized = normalizeTaxonomyValue(value);
+    if (!normalized) continue;
+
+    if (seen.has(normalized)) continue;
+    seen.add(normalized);
+    result.push(normalized);
+  }
+
+  return result;
+}

src/verify/providers/openfoodfacts/open-foodfacts.provider.spec.ts

@@ -92,7 +92,7 @@ describe('OpenFoodFactsProvider', () => {
     const result = await provider.lookupProduct('123');
 
     expect(result.ingredients).toEqual(['sugar', 'salt', 'pepper']);
-    expect(result.allergens).toEqual(['en:milk']);
+    expect(result.allergens).toEqual(['milk']);
   });
 
   it('falls back to ingredients_tags when ingredients_text is empty', async () => {
@@ -118,7 +118,7 @@ describe('OpenFoodFactsProvider', () => {
     );
 
     const result = await provider.lookupProduct('123');
-    expect(result.ingredients).toEqual(['en:sugar', 'en:salt']);
+    expect(result.ingredients).toEqual(['sugar', 'salt']);
   });
 
   it('throws NotFoundException for 404 responses', async () => {
@@ -148,4 +148,3 @@ describe('OpenFoodFactsProvider', () => {
     );
   });
 });
-

src/verify/providers/openfoodfacts/open-foodfacts.provider.ts

@@ -8,6 +8,8 @@ import {
   ServiceUnavailableException,
 } from '@nestjs/common';
 import { firstValueFrom } from 'rxjs';
+import { normalizeTaxonomyArray } from '../../../common/utils/taxonomy-normalizer.util';
+import { NormalizedProduct } from '../region-inference-provider/interface/normalized-product.interface';
 
 type OpenFoodFactsProductResponse = {
   status?: 0 | 1;
@@ -31,24 +33,15 @@ type OpenFoodFactsProductResponse = {
   };
 };
 
-export interface NormalizedProduct {
-  barcode: string;
-  name?: string;
-  brand?: string;
+export interface OpenFoodFactsNormalizedProduct extends NormalizedProduct {
   quantity?: string;
   imageUrl?: string;
   nutriments?: Record<string, unknown>;
-  ingredients: string[];
   allergens: string[];
-  traces: string[];
-  manufacturingCountries: string[];
-  purchaseCountries: string[];
-  languages: string[];
-  labels: string[];
   rawSource: 'openfoodfacts';
 }
 
-export type OpenFoodFactsLookupResult = NormalizedProduct;
+export type OpenFoodFactsLookupResult = OpenFoodFactsNormalizedProduct;
 
 function asStringArray(value: unknown): string[] {
   if (!Array.isArray(value)) return [];
@@ -72,6 +65,28 @@ export class OpenFoodFactsProvider {
     private readonly configService: ConfigService,
   ) {}
 
+  private summarizeArray(values: string[]): { count: number; sample: string[] } {
+    const sampleLimit = 5;
+    return { count: values.length, sample: values.slice(0, sampleLimit) };
+  }
+
+  private logNormalization(
+    code: string,
+    field: string,
+    before: string[],
+    after: string[],
+  ): void {
+    this.logger.debug(
+      JSON.stringify({
+      msg: 'taxonomy_normalization',
+      code,
+      field,
+      before: this.summarizeArray(before),
+      after: this.summarizeArray(after),
+      }),
+    );
+  }
+
   async lookupProduct(code: string): Promise<OpenFoodFactsLookupResult> {
     const baseUrl =
       this.configService.get<string>('OPENFOODFACTS_BASE_URL') ??
@@ -127,21 +142,56 @@ export class OpenFoodFactsProvider {
     const ingredientsFromText = splitIngredientsText(product.ingredients_text);
     const ingredientsFromTags = asStringArray(product.ingredients_tags);
 
+    const rawAllergens = asStringArray(product.allergens_tags);
+    const rawTraces = asStringArray(product.traces_tags);
+    const rawManufacturingCountries = asStringArray(product.manufacturing_places_tags);
+    const rawPurchaseCountries = asStringArray(product.countries_tags);
+    const rawLanguages = asStringArray(product.languages_hierarchy);
+    const rawLabels = asStringArray(product.labels_tags);
+    const rawIngredients =
+      ingredientsFromText.length > 0 ? ingredientsFromText : ingredientsFromTags;
+
+    const allergens = normalizeTaxonomyArray(rawAllergens);
+    const traces = normalizeTaxonomyArray(rawTraces);
+    const manufacturingCountries = normalizeTaxonomyArray(rawManufacturingCountries);
+    const purchaseCountries = normalizeTaxonomyArray(rawPurchaseCountries);
+    const languages = normalizeTaxonomyArray(rawLanguages);
+    const labels = normalizeTaxonomyArray(rawLabels);
+
+    const ingredients =
+      ingredientsFromText.length > 0
+        ? rawIngredients
+        : normalizeTaxonomyArray(rawIngredients);
+
+    this.logNormalization(code, 'allergens', rawAllergens, allergens);
+    this.logNormalization(code, 'traces', rawTraces, traces);
+    this.logNormalization(
+      code,
+      'manufacturingCountries',
+      rawManufacturingCountries,
+      manufacturingCountries,
+    );
+    this.logNormalization(code, 'purchaseCountries', rawPurchaseCountries, purchaseCountries);
+    this.logNormalization(code, 'languages', rawLanguages, languages);
+    this.logNormalization(code, 'labels', rawLabels, labels);
+    if (ingredientsFromText.length === 0) {
+      this.logNormalization(code, 'ingredients', rawIngredients, ingredients);
+    }
+
     return {
       barcode: code,
       name: product.product_name,
       brand: product.brands,
       quantity: product.quantity,
       imageUrl: product.image_front_url ?? product.image_url,
       nutriments: product.nutriments,
-      ingredients:
-        ingredientsFromText.length > 0 ? ingredientsFromText : ingredientsFromTags,
-      allergens: asStringArray(product.allergens_tags),
-      traces: asStringArray(product.traces_tags),
-      manufacturingCountries: asStringArray(product.manufacturing_places_tags),
-      purchaseCountries: asStringArray(product.countries_tags),
-      languages: asStringArray(product.languages_hierarchy),
-      labels: asStringArray(product.labels_tags),
+      ingredients,
+      allergens,
+      traces,
+      manufacturingCountries,
+      purchaseCountries,
+      languages,
+      labels,
       rawSource: 'openfoodfacts',
     };
   }

src/verify/providers/region-inference-provider/interface/normalized-product.interface.ts

@@ -1,19 +1,11 @@
 export interface NormalizedProduct {
   barcode: string;
-
   name?: string;
-
   brand?: string;
-
   manufacturingCountries: string[];
-
   purchaseCountries: string[];
-
   languages: string[];
-
   labels: string[];
-
   traces: string[];
-
   ingredients: string[];
 }

src/verify/providers/region-inference-provider/region-inference.provider.spec.ts

@@ -13,8 +13,8 @@ describe('RegionInferenceProvider', () => {
       barcode: '123',
       name: 'Test Product',
       brand: 'Test Brand',
-      manufacturingCountries: [' Nigeria '],
-      purchaseCountries: ['USA'],
+      manufacturingCountries: ['nigeria'],
+      purchaseCountries: ['usa'],
       languages: [],
       labels: [],
       traces: [],
@@ -31,12 +31,12 @@ describe('RegionInferenceProvider', () => {
         expect.objectContaining({
           region: Region.NIGERIA,
           source: 'manufacturing_country',
-          matchedValue: ' Nigeria ',
+          matchedValue: 'nigeria',
         }),
         expect.objectContaining({
           region: Region.USA,
           source: 'purchase_country',
-          matchedValue: 'USA',
+          matchedValue: 'usa',
         }),
       ]),
     );
@@ -45,7 +45,7 @@ describe('RegionInferenceProvider', () => {
   it('returns empty confidence when no signals match', () => {
     const result = provider.infer({
       barcode: '123',
-      manufacturingCountries: ['NowhereLand'],
+      manufacturingCountries: ['nowhereland'],
       purchaseCountries: [],
       languages: [],
       labels: [],
@@ -57,4 +57,3 @@ describe('RegionInferenceProvider', () => {
     expect(result.evidence).toEqual([]);
   });
 });
-

src/verify/providers/region-inference-provider/region-inference.provider.ts

@@ -1,16 +1,12 @@
 import { Injectable, Logger } from '@nestjs/common';
 import { COUNTRY_RULES } from './rules';
-// import { BARCODE_PREFIX_TO_REGION_RULES } from './rules';
 import { Region } from './region.enums';
 import { InferenceEvidence, InferenceResult, NormalizedProduct } from './interface';
 import { INFERENCE_WEIGHTS, MINIMUM_CONFIDENCE_THRESHOLD } from './region.constants';
 import {
-  // InferenceScores,
   RegionConfidenceMap,
-  // EvidenceMatch,
 } from './region.types';
 
-
 @Injectable()
 export class RegionInferenceProvider {
   private readonly logger = new Logger(
@@ -21,7 +17,6 @@ export class RegionInferenceProvider {
     product: NormalizedProduct,
   ): InferenceResult {
     const scores: RegionConfidenceMap = {};
-
     const evidence: InferenceEvidence[] = [];
 
     this.inferManufacturing(
@@ -51,14 +46,11 @@ export class RegionInferenceProvider {
     evidence: InferenceEvidence[],
   ): void {
     for (const country of product.manufacturingCountries) {
-      const normalized =
-        this.normalize(country);
-
       for (const [region, keywords] of Object.entries(
         COUNTRY_RULES,
       )) {
         if (
-          keywords.includes(normalized)
+          keywords.includes(country)
         ) {
           this.addScore(
             scores,
@@ -85,14 +77,11 @@ export class RegionInferenceProvider {
     evidence: InferenceEvidence[],
   ): void {
     for (const country of product.purchaseCountries) {
-      const normalized =
-        this.normalize(country);
-
       for (const [region, keywords] of Object.entries(
         COUNTRY_RULES,
       )) {
         if (
-          keywords.includes(normalized)
+          keywords.includes(country)
         ) {
           this.addScore(
             scores,
@@ -153,12 +142,4 @@ export class RegionInferenceProvider {
 
     return normalized;
   }
-
-  private normalize(
-    value: string,
-  ): string {
-    return value
-      .trim()
-      .toLowerCase();
-  }
-}
+}