Sub task/setup the infrastructure and backend interfaces core 555

backend/package.json

@@ -8,8 +8,8 @@
     "clean": "rimraf -I build coverage deploy",
     "compile": "tsc --build tsconfig.prod.json",
     "prebuild": "npm run clean && npm run compile",
-    "build": "esbuild ./src/index.ts ./src/import/async/index.ts ./src/export/async/index.ts ./src/auth/authenticator/index.ts --bundle --tsconfig=tsconfig.prod.json --sourcemap --minify --platform=node --target=node16 --outdir=build --main-fields=module,main",
-    "postbuild": "cp ./package.json ./build/ && (cd ./build; npm pkg delete devDependencies; npm pkg delete dependencies; npm pkg delete scripts) && cp ./build/package.json ./build/import/async && mkdir build/rest && mv build/index.js build/index.js.map build/package.json ./build/rest/",
+    "build": "esbuild ./src/index.ts ./src/import/async/index.ts ./src/export/async/index.ts ./src/embeddings/handler/index.ts ./src/auth/authenticator/index.ts --bundle --tsconfig=tsconfig.prod.json --sourcemap --minify --platform=node --target=node16 --outdir=build --main-fields=module,main",
+    "postbuild": "cp ./package.json ./build/ && (cd ./build; npm pkg delete devDependencies; npm pkg delete dependencies; npm pkg delete scripts) && cp ./build/package.json ./build/import/async && cp ./build/package.json ./build/embeddings/handler && mkdir build/rest && mv build/index.js build/index.js.map build/package.json ./build/rest/",
     "test": "jest --coverage",
     "test:integration": "jest --config jest.integration.config.js --runInBand",
     "test:smoke": "jest --config jest.smoke.config.js --runInBand test/smoke/*.test.ts",

backend/src/_test_utilities/getTestConfiguration.ts

@@ -13,5 +13,9 @@ export function getTestConfiguration(dbname: string): IConfiguration {
     asyncImportLambdaFunctionArn: "arn:aws:lambda:foo:bar:baz:import",
     asyncExportLambdaFunctionArn: "arn:aws:lambda:foo:bar:baz:export",
     asyncLambdaFunctionRegion: "foo",
+    geminiApiKey: "test-gemini-api-key",
+    geminiEmbeddingModel: "text-embedding-004",
+    embeddingsQueueUrl: "https://sqs.foo.amazonaws.com/123456789012/test-embeddings-queue",
+    embeddingsQueueRegion: "foo",
   };
 }

backend/src/common/lambda.types.ts

@@ -6,4 +6,5 @@ export enum Lambdas {
   API = "API",
   IMPORT = "IMPORT",
   EXPORT = "EXPORT",
+  EMBEDDING = "EMBEDDING",
 }

backend/src/embeddings/handler/index.ts

@@ -0,0 +1,50 @@
+import { SQSEvent, SQSRecord } from "aws-lambda";
+import { initOnce as serverInitOnce } from "server/init";
+import { initializeSentry } from "initializeSentry";
+import * as Sentry from "@sentry/aws-serverless";
+import { Lambdas } from "common/lambda.types";
+import { IGenerateEmbeddingTask } from "embeddings/service/types";
+import { EmbeddingService } from "embeddings/service/service";
+
+initializeSentry(Lambdas.EMBEDDING);
+
+async function initOnce() {
+  await serverInitOnce();
+  return new EmbeddingService();
+}
+
+/**
+ * Entry point for the embeddings lambda function.
+ *
+ * This lambda is triggered by the embeddings SQS queue via an event source mapping: when N messages
+ * are available on the queue, the lambda is invoked with up to N records in `event.Records`, and this
+ * handler generates the embeddings for each of them in the background.
+ *
+ * If a record throws, we rethrow so that AWS retries the message and, after the configured number of
+ * attempts, moves it to the dead-letter queue.
+ */
+export const handler = Sentry.wrapHandler(async (event: SQSEvent): Promise<void> => {
+  const records = event.Records ?? [];
+  console.info(`Embeddings lambda triggered with ${records.length} record(s)`);
+
+  // Initialize the connection to the database (and registries). If it fails, rethrow so the lambda is retried.
+  const embeddingService = await initOnce();
+
+  for (const record of records) {
+    await handleRecord(embeddingService, record);
+  }
+});
+
+async function handleRecord(embeddingService: EmbeddingService, record: SQSRecord): Promise<void> {
+  let task: IGenerateEmbeddingTask;
+  try {
+    task = JSON.parse(record.body) as IGenerateEmbeddingTask;
+  } catch (e: unknown) {
+    // A malformed message will never succeed on retry, so log and skip it rather than throwing.
+    console.error(new Error(`Embeddings lambda: skipping unparseable SQS record ${record.messageId}`, { cause: e }));
+    return;
+  }
+
+  console.info("Processing embedding task", { messageId: record.messageId, task });
+  await embeddingService.processTask(task);
+}

backend/src/embeddings/models/gemini/geminiService.ts

@@ -0,0 +1,43 @@
+import { IEmbeddingModelService } from "embeddings/models/modelsServiceTypes";
+
+/**
+ * Base URL of the Gemini (Generative Language) REST API.
+ */
+// const GEMINI_API_BASE_URL = "https://generativelanguage.googleapis.com/v1beta";
+
+/**
+ * GeminiService talks to the Gemini REST API directly via `fetch`
+ * (no SDK dependency, so it bundles cleanly with esbuild).
+ *
+ * See:
+ * - https://ai.google.dev/api/embeddings#method:-models.embedcontent
+ * - https://ai.google.dev/api/embeddings#method:-models.batchembedcontents
+ */
+export class GeminiService implements IEmbeddingModelService {
+  private readonly apiKey: string;
+  private readonly model: string;
+
+  constructor(apiKey: string, model: string) {
+    if (!apiKey) {
+      throw new Error("GeminiService: GEMINI_API_KEY is not configured");
+    }
+    this.apiKey = apiKey;
+
+    if (!model)
+      throw new Error(
+        "GeminiService: GEMINI_MODEL_NAME is not configured. Please set it in the environment variables or in the config file."
+      );
+    this.model = model;
+  }
+
+  async generateEmbeddingBatch(texts: string[]): Promise<number[][]> {
+    console.debug("GeminiService: generateEmbeddingBatch", { texts });
+    console.warn("Not implemented yet");
+    return [[]];
+  }
+
+  async generateEmbedding(text: string): Promise<number[]> {
+    const [vector] = await this.generateEmbeddingBatch([text]);
+    return vector;
+  }
+}

backend/src/embeddings/models/modelsServiceTypes.ts

@@ -0,0 +1,16 @@
+export interface IEmbeddingModelService {
+  /**
+   * Generate an embedding vector for the given text.
+   *
+   * @param text - The text to embed.
+   * @returns The embedding vector as an array of numbers.
+   * @throws If the API key is not configured, or the API  returns an error / empty response.
+   */
+  generateEmbedding(text: string): Promise<number[]>;
+
+  /**
+   * Generate embeddings for a batch of texts.
+   * @param texts
+   */
+  generateEmbeddingBatch(texts: string[]): Promise<number[][]>;
+}

backend/src/embeddings/service/client.ts

@@ -0,0 +1,12 @@
+import { IGenerateEmbeddingTask } from "./types";
+
+export interface IEmbeddingClient {
+  pushTaskToQueue(task: IGenerateEmbeddingTask): Promise<void>;
+}
+
+export class EmbeddingClient implements IEmbeddingClient {
+  pushTaskToQueue(task: IGenerateEmbeddingTask): Promise<void> {
+    console.debug("EmbeddingClient.pushTaskToQueue", task);
+    throw new Error("Method not implemented.");
+  }
+}

backend/src/embeddings/service/service.ts

@@ -0,0 +1,17 @@
+import { IGenerateEmbeddingTask } from "./types";
+
+export interface IEmbeddingService {
+  processTask(task: IGenerateEmbeddingTask): Promise<void>;
+}
+
+export class EmbeddingService implements IEmbeddingService {
+  constructor() {}
+
+  async processTask(task: IGenerateEmbeddingTask): Promise<void> {
+    console.info("Generated embedding", {
+      modelId: task.modelId,
+      entityType: task.entityType,
+      entityId: task.entityId,
+    });
+  }
+}

backend/src/embeddings/service/types.ts

@@ -0,0 +1,20 @@
+export enum EmbeddableEntityType {
+  Skill = "Skill",
+  Occupation = "Occupation",
+  OccupationGroup = "OccupationGroup",
+  SkillGroup = "SkillGroup",
+}
+
+export enum EmbeddableField {
+  preferredLabel = "preferredLabel",
+  description = "description",
+  altLabels = "altLabels",
+  scopeNote = "scopeNote",
+}
+
+export interface IGenerateEmbeddingTask {
+  modelId: string;
+  entityId: string;
+  entityType: EmbeddableEntityType;
+  fields: EmbeddableField;
+}

backend/src/index.ts

@@ -1,6 +1,7 @@
 import { APIGatewayProxyEvent, Handler } from "aws-lambda";
 import { handler as InfoHandler } from "applicationInfo";
 import { handler as ModelHandler } from "modelInfo";
+import { handler as ModelEmbeddingProcessesHandler } from "modelInfo/embeddingProcesses";
 import { handler as ImportHandler } from "import";
 import { handler as OccupationGroupHandler } from "esco/occupationGroup/index";
 import { handler as OccupationHandler } from "esco/occupations";
@@ -65,6 +66,8 @@ export const handleRouteEvent = async (event: APIGatewayProxyEvent) => {
     return ImportHandler(event);
   } else if (path === Routes.EXPORT_ROUTE) {
     return ExportHandler(event);
+  } else if (pathToRegexp(Routes.MODEL_EMBEDDING_PROCESSES_ROUTE).regexp.test(path)) {
+    return ModelEmbeddingProcessesHandler(event);
   } else if (
     pathToRegexp([
       Routes.OCCUPATION_GROUPS_ROUTE,

backend/src/modelInfo/embeddingProcesses/POST/index.ts

@@ -0,0 +1,8 @@
+import { APIGatewayProxyEvent, APIGatewayProxyResult } from "aws-lambda";
+
+export class POSTModelEmbeddingProcessesHandler {
+  async handle(_event: APIGatewayProxyEvent): Promise<APIGatewayProxyResult> {
+    console.log(_event);
+    throw new Error("Not implemented yet");
+  }
+}

backend/src/modelInfo/embeddingProcesses/index.ts

@@ -0,0 +1,15 @@
+import { APIGatewayProxyEvent } from "aws-lambda";
+import { APIGatewayProxyResult } from "aws-lambda/trigger/api-gateway-proxy";
+import { HTTP_VERBS, STD_ERRORS_RESPONSES } from "server/httpUtils";
+import { POSTModelEmbeddingProcessesHandler } from "./POST";
+
+export const handler: (
+  event: APIGatewayProxyEvent /*, context: Context, callback: Callback*/
+) => Promise<APIGatewayProxyResult> = async (
+  event: APIGatewayProxyEvent /*, context: Context, callback: Callback*/
+) => {
+  if (event?.httpMethod === HTTP_VERBS.POST) {
+    return new POSTModelEmbeddingProcessesHandler().handle(event);
+  }
+  return STD_ERRORS_RESPONSES.METHOD_NOT_ALLOWED;
+};

backend/src/routes.constant.ts

@@ -27,4 +27,5 @@ export const Routes = {
   SKILL_OCCUPATIONS_ROUTE: "/models/:modelId/skills/:id/occupations",
   SKILL_RELATED_ROUTE: "/models/:modelId/skills/:id/related",
   SKILL_HISTORY_ROUTE: "/models/:modelId/skills/:id/history",
+  MODEL_EMBEDDING_PROCESSES_ROUTE: "/models/:modelId/embedding-processes",
 };

backend/src/server/config/config.test.ts

@@ -47,6 +47,10 @@ describe("Test read Configuration()", () => {
     process.env.ASYNC_IMPORT_LAMBDA_FUNCTION_ARN = getRandomString(10);
     process.env.ASYNC_EXPORT_LAMBDA_FUNCTION_ARN = getRandomString(10);
     process.env.ASYNC_LAMBDA_FUNCTION_REGION = getRandomString(10);
+    process.env.GEMINI_API_KEY = getRandomString(10);
+    process.env.GEMINI_EMBEDDING_MODEL = getRandomString(10);
+    process.env.EMBEDDINGS_QUEUE_URL = getRandomString(10);
+    process.env.EMBEDDINGS_QUEUE_REGION = getRandomString(10);
 
     // WHEN reading the configuration from the environment
     const actualConfig = readEnvironmentConfiguration();
@@ -63,6 +67,10 @@ describe("Test read Configuration()", () => {
       asyncImportLambdaFunctionArn: process.env.ASYNC_IMPORT_LAMBDA_FUNCTION_ARN,
       asyncExportLambdaFunctionArn: process.env.ASYNC_EXPORT_LAMBDA_FUNCTION_ARN,
       asyncLambdaFunctionRegion: process.env.ASYNC_LAMBDA_FUNCTION_REGION,
+      geminiApiKey: process.env.GEMINI_API_KEY,
+      geminiEmbeddingModel: process.env.GEMINI_EMBEDDING_MODEL,
+      embeddingsQueueUrl: process.env.EMBEDDINGS_QUEUE_URL,
+      embeddingsQueueRegion: process.env.EMBEDDINGS_QUEUE_REGION,
     });
   });
 
@@ -77,6 +85,10 @@ describe("Test read Configuration()", () => {
     delete process.env.DOWNLOAD_BUCKET_REGION;
     delete process.env.ASYNC_LAMBDA_FUNCTION_ARN;
     delete process.env.ASYNC_LAMBDA_FUNCTION_REGION;
+    delete process.env.GEMINI_API_KEY;
+    delete process.env.GEMINI_EMBEDDING_MODEL;
+    delete process.env.EMBEDDINGS_QUEUE_URL;
+    delete process.env.EMBEDDINGS_QUEUE_REGION;
 
     // WHEN reading the configuration from the environment
     const config = readEnvironmentConfiguration();
@@ -93,6 +105,10 @@ describe("Test read Configuration()", () => {
       asyncImportLambdaFunctionArn: "",
       asyncExportLambdaFunctionArn: "",
       asyncLambdaFunctionRegion: "",
+      geminiApiKey: "",
+      geminiEmbeddingModel: "",
+      embeddingsQueueUrl: "",
+      embeddingsQueueRegion: "",
     });
   });
 });
@@ -189,5 +205,9 @@ function getMockConfig(): IConfiguration {
     asyncImportLambdaFunctionArn: getTestString(10),
     asyncExportLambdaFunctionArn: getTestString(10),
     asyncLambdaFunctionRegion: getTestString(10),
+    geminiApiKey: getTestString(10),
+    geminiEmbeddingModel: getTestString(10),
+    embeddingsQueueRegion: getTestString(10),
+    embeddingsQueueUrl: getTestString(10),
   };
 }

backend/src/server/config/config.ts

@@ -11,6 +11,10 @@ export const ENV_VAR_NAMES = {
   ASYNC_IMPORT_LAMBDA_FUNCTION_ARN: "ASYNC_IMPORT_LAMBDA_FUNCTION_ARN",
   ASYNC_EXPORT_LAMBDA_FUNCTION_ARN: "ASYNC_EXPORT_LAMBDA_FUNCTION_ARN",
   ASYNC_LAMBDA_FUNCTION_REGION: "ASYNC_LAMBDA_FUNCTION_REGION",
+  GEMINI_API_KEY: "GEMINI_API_KEY",
+  GEMINI_EMBEDDING_MODEL: "GEMINI_EMBEDDING_MODEL",
+  EMBEDDINGS_QUEUE_URL: "EMBEDDINGS_QUEUE_URL",
+  EMBEDDINGS_QUEUE_REGION: "EMBEDDINGS_QUEUE_REGION",
 };
 
 export interface IConfiguration {
@@ -24,6 +28,10 @@ export interface IConfiguration {
   asyncImportLambdaFunctionArn: string;
   asyncExportLambdaFunctionArn: string;
   asyncLambdaFunctionRegion: string;
+  geminiApiKey: string;
+  geminiEmbeddingModel: string;
+  embeddingsQueueUrl: string;
+  embeddingsQueueRegion: string;
 }
 export function readEnvironmentConfiguration(): IConfiguration {
   return {
@@ -37,6 +45,10 @@ export function readEnvironmentConfiguration(): IConfiguration {
     asyncImportLambdaFunctionArn: process.env[ENV_VAR_NAMES.ASYNC_IMPORT_LAMBDA_FUNCTION_ARN] ?? "",
     asyncExportLambdaFunctionArn: process.env[ENV_VAR_NAMES.ASYNC_EXPORT_LAMBDA_FUNCTION_ARN] ?? "",
     asyncLambdaFunctionRegion: process.env[ENV_VAR_NAMES.ASYNC_LAMBDA_FUNCTION_REGION] ?? "",
+    geminiApiKey: process.env[ENV_VAR_NAMES.GEMINI_API_KEY] ?? "",
+    geminiEmbeddingModel: process.env[ENV_VAR_NAMES.GEMINI_EMBEDDING_MODEL] ?? "",
+    embeddingsQueueUrl: process.env[ENV_VAR_NAMES.EMBEDDINGS_QUEUE_URL] ?? "",
+    embeddingsQueueRegion: process.env[ENV_VAR_NAMES.EMBEDDINGS_QUEUE_REGION] ?? "",
   };
 }
 
@@ -88,3 +100,19 @@ export function getAsyncLambdaFunctionRegion() {
 export function getDomainName() {
   return _configuration?.domainName ?? "";
 }
+
+export function getGeminiApiKey() {
+  return _configuration?.geminiApiKey ?? "";
+}
+
+export function getGeminiEmbeddingModel() {
+  return _configuration?.geminiEmbeddingModel ?? "";
+}
+
+export function getEmbeddingsQueueUrl() {
+  return _configuration?.embeddingsQueueUrl ?? "";
+}
+
+export function getEmbeddingsQueueRegion() {
+  return _configuration?.embeddingsQueueRegion ?? "";
+}