fix: keep non Latin letters when normalizing person names

src/lib/people/normalization.ts

@@ -10,8 +10,14 @@ export function normalizeEmail(email?: string | null) {
   return value;
 }
 
+/** Lowercases, drops accents and keeps letters from every script, so Cyrillic or CJK names never collapse to "". */
 export function normalizedName(name: string) {
-  return name.toLowerCase().replace(/[^a-z0-9]+/g, " ").trim();
+  return name
+    .normalize("NFKD")
+    .replace(/\p{M}+/gu, "")
+    .toLowerCase()
+    .replace(/[^\p{L}\p{N}]+/gu, " ")
+    .trim();
 }
 
 export function canonicalUrl(url: string) {

src/lib/people/openai.ts

@@ -20,10 +20,9 @@ import {
   expandStageTerms,
   isInvestmentPersonType,
   normalizePersonTypes,
-  normalizeTerm,
   splitListInput,
 } from "./search-taxonomy";
-import { domainFromUrlOrEmail } from "./normalization";
+import { domainFromUrlOrEmail, normalizedName } from "./normalization";
 
 const OPENAI_RESPONSES_URL = "https://api.openai.com/v1/responses";
 const DEFAULT_MODEL = "gpt-4o-mini";
@@ -738,7 +737,7 @@ function parseOrganizations(json: unknown): CandidateParseResult<PeopleProviderO
 function parsePeople(json: unknown, organizations: PeopleProviderOrganization[], input: PeopleProviderSearchInput): CandidateParseResult<PeopleProviderPerson> {
   const root = z.object({ people: z.array(z.unknown()).default([]), candidates: z.array(z.unknown()).default([]) }).passthrough().safeParse(json);
   const raw = root.success ? [...root.data.people, ...root.data.candidates] : [];
-  const organizationByName = new Map(organizations.map((organization) => [normalizeTerm(organization.name), organization]));
+  const organizationByName = new Map(organizations.map((organization) => [normalizedName(organization.name), organization]));
   const values: PeopleProviderPerson[] = [];
   const rejections: PeopleSearchRejection[] = [];
   let validNames = 0;
@@ -772,7 +771,7 @@ function normalizeRawPerson(raw: unknown, organizationByName: Map<string, People
   const optionalFields = objectRecord(data.optionalSourceSupportedFields);
   const rawOrganization = objectRecord(data.currentOrganization ?? data.organization);
   const organizationName = stringValue(data.currentOrganizationName ?? data.organizationName ?? data.firm ?? data.fund ?? rawOrganization.name ?? (typeof data.organization === "string" ? data.organization : null));
-  const currentOrganization = organizationName ? organizationByName.get(normalizeTerm(organizationName)) ?? normalizeRawOrganization(rawOrganization.name ? rawOrganization : { name: organizationName }) : null;
+  const currentOrganization = organizationName ? organizationByName.get(normalizedName(organizationName)) ?? normalizeRawOrganization(rawOrganization.name ? rawOrganization : { name: organizationName }) : null;
   const currentTitle = stringValue(data.currentTitle ?? data.title ?? data.role);
   const explicitTypes = normalizePersonTypes(data.personTypes ?? data.personType ?? data.type);
   const personTypes = explicitTypes.length ? explicitTypes : isInvestmentPersonType([], currentTitle) ? [PersonType.INVESTOR] : [];
@@ -1042,8 +1041,8 @@ function dedupeProviderPeople(result: CandidateParseResult<PeopleProviderPerson>
 }
 
 function providerPersonKey(person: PeopleProviderPerson) {
-  const name = normalizeTerm(person.fullName);
-  const organization = normalizeTerm(person.currentOrganizationName ?? "");
+  const name = normalizedName(person.fullName);
+  const organization = normalizedName(person.currentOrganizationName ?? "");
   const linkedin = person.linkedinUrl ? canonicalUrl(person.linkedinUrl) : "";
   return [name, organization, linkedin].filter(Boolean).join("|");
 }

tests/people-search-pipeline.test.ts

@@ -1,6 +1,8 @@
 import { describe, expect, it } from "vitest";
 import { PersonType, type DiscoveredPerson, type PersonRelationshipEnrichment } from "@prisma/client";
 import { buildPeopleDiscoveryBatchesForTest, buildPeopleResearchQueries, dedupeProviderPeopleForTest, parsePeopleDiscoveryJsonForTest, summarizeOpenAIResponseForTest } from "@/lib/people/openai";
+import { fingerprintPerson } from "@/lib/people/entity-resolution";
+import { normalizedName } from "@/lib/people/normalization";
 import { interpretPeopleSearchObjective } from "@/lib/people/query";
 import { calculatePeopleFitScore } from "@/lib/people/scoring";
 import { evaluateHardFilters, searchOnlyStartupContext } from "@/lib/people/search";
@@ -226,6 +228,27 @@ describe("external people search pipeline", () => {
     expect(deduped.values.map((person) => person.fullName)).toEqual(["Alice Morgan", "Brian Shah"]);
     expect(deduped.rejections).toHaveLength(0);
   });
+
+  it("keeps distinct people whose names use a non Latin script", () => {
+    const parsed = parsePeopleDiscoveryJsonForTest({
+      people: [
+        sourceBackedRawInvestor({ fullName: "Олена Коваль", currentOrganizationName: "Frontier Defence Ventures" }),
+        sourceBackedRawInvestor({ fullName: "Іван Петренко", currentOrganizationName: "Frontier Defence Ventures" }),
+        sourceBackedRawInvestor({ fullName: "Олена Коваль", currentOrganizationName: "Frontier Defence Ventures" }),
+      ],
+    }, providerInput());
+    const deduped = dedupeProviderPeopleForTest(parsed);
+
+    expect(deduped.values.map((person) => person.fullName)).toEqual(["Олена Коваль", "Іван Петренко"]);
+    expect(deduped.rejections).toEqual([expect.objectContaining({ candidate: "Олена Коваль", reasons: ["duplicate_identity"] })]);
+  });
+
+  it("normalizes names from every script without dropping letters", () => {
+    expect(normalizedName("Олена  Коваль")).toBe("олена коваль");
+    expect(normalizedName("José García")).toBe("jose garcia");
+    expect(normalizedName("Іван Петренко")).not.toBe(normalizedName("Олена Коваль"));
+    expect(fingerprintPerson({ fullName: "Олена Коваль", currentOrganizationName: "Fund" })).toBe("олена коваль|fund");
+  });
 });
 
 function screenshotSearchInput(overrides: Record<string, unknown> = {}) {