ci(AIC-3417): test the published packages in a real AWS Lambda [4/4]

.github/workflows/release-please.yml

@@ -10,6 +10,11 @@
 #   - Allows publishing a specific package to npm
 #   - Supports prerelease and dry-run modes
 #
+# RELEASE CANARY (after a publish, or on demand):
+#   - Installs the exact published versions into a CommonJS Serverless/Webpack app
+#   - Deploys, invokes, and removes an ephemeral Node 22 Lambda via GitHub OIDC
+#   - Requires the AWS_LAMBDA_CANARY_ROLE_ARN repository variable; skipped while it is unset
+#
 name: release-please
 
 on:
@@ -41,6 +46,11 @@ on:
         description: 'Is this a dry run. If so no package will be published.'
         type: boolean
         required: true
+      canary_versions:
+        description: 'Run only the Lambda release canary against these exact published versions, e.g. ai-server=0.4.0,ai-node=0.3.0,openai-messages=0.3.0,ai-otel=0.2.0'
+        type: string
+        required: false
+        default: ''
 
 jobs:
   release-please:
@@ -303,7 +313,7 @@ jobs:
 
   manual-publish:
     runs-on: ubuntu-latest
-    if: github.event_name == 'workflow_dispatch'
+    if: ${{ github.event_name == 'workflow_dispatch' && inputs.canary_versions == '' }}
     permissions:
       id-token: write
       contents: read
@@ -319,3 +329,123 @@ jobs:
           workspace_path: ${{ inputs.workspace_path }}
           prerelease: ${{ inputs.prerelease }}
           dry_run: ${{ inputs.dry_run }}
+
+  # --- Published-artifact Lambda release canary ---
+  #
+  # This job installs the versions that npm now serves into a clean CommonJS package.
+  # Serverless 3, Webpack 5, and webpack-node-externals 3 bundle that package.
+  # A real Node 22 Lambda then loads it, and the job deletes the Lambda again.
+  # The job loads packages only. It uses no SDK key, no flag evaluation, and no model call.
+  # See integration-tests/lambda-commonjs/README.md for the AWS role that this job needs.
+
+  release-canary:
+    name: Published-artifact Lambda canary
+    runs-on: ubuntu-latest
+    needs:
+      - release-please
+      - release-client
+      - release-ai-node
+      - release-ai-otel
+      - release-openai-messages
+    if: >-
+      ${{ always()
+        && !contains(needs.*.result, 'failure')
+        && !contains(needs.*.result, 'cancelled')
+        && vars.AWS_LAMBDA_CANARY_ROLE_ARN != ''
+        && vars.AWS_LAMBDA_CANARY_PERMISSIONS_BOUNDARY != ''
+        && (github.event_name != 'workflow_dispatch' || inputs.canary_versions != '')
+        && (github.event_name == 'workflow_dispatch'
+          || needs.release-please.outputs.package-client-released == 'true'
+          || needs.release-please.outputs.package-ai-node-released == 'true'
+          || needs.release-please.outputs.package-ai-otel-released == 'true'
+          || needs.release-please.outputs.package-openai-messages-released == 'true') }}
+    permissions:
+      contents: read
+      id-token: write
+    env:
+      AWS_REGION: ${{ vars.AWS_LAMBDA_CANARY_REGION || 'us-east-1' }}
+      CANARY_STAGE: c${{ github.run_id }}-${{ github.run_attempt }}
+      SLS_TELEMETRY_DISABLED: '1'
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+
+      - name: Resolve canary versions
+        id: versions
+        env:
+          REQUESTED_VERSIONS: ${{ inputs.canary_versions }}
+        run: |
+          set -euo pipefail
+          if [ -n "${REQUESTED_VERSIONS:-}" ]; then
+            resolved="$REQUESTED_VERSIONS"
+          else
+            resolved=$(node -e "const m=require('./.release-please-manifest.json');const d={'ai-server':'packages/client','ai-node':'packages/ai-node','openai-messages':'packages/openai-messages','ai-otel':'packages/ai-otel'};process.stdout.write(Object.entries(d).map(([n,p])=>n+'='+m[p]).join(','))")
+          fi
+          echo "Canary versions: $resolved"
+          echo "resolved=$resolved" >> "$GITHUB_OUTPUT"
+
+      - name: Wait for the versions to be visible on npm
+        env:
+          LD_AI_CANARY_VERSIONS: ${{ steps.versions.outputs.resolved }}
+        run: node integration-tests/lambda-commonjs/canary.mjs wait
+
+      # The build runs npm install for the Serverless toolchain, and every install script
+      # inherits this environment. The build needs no AWS access, so the credentials
+      # start after it.
+      - name: Build the canary package
+        env:
+          LD_AI_CANARY_VERSIONS: ${{ steps.versions.outputs.resolved }}
+        run: node integration-tests/lambda-commonjs/canary.mjs build --package --stage="$CANARY_STAGE"
+
+      - name: Configure AWS credentials
+        id: aws-credentials
+        uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1
+        with:
+          audience: 'https://github.com/launchdarkly'
+          role-to-assume: ${{ vars.AWS_LAMBDA_CANARY_ROLE_ARN }}
+          role-session-name: JsAiSdkLambdaCanary
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Deploy the ephemeral Lambda
+        working-directory: integration-tests/lambda-commonjs/.canary/app
+        env:
+          LD_AI_CANARY_PERMISSIONS_BOUNDARY: ${{ vars.AWS_LAMBDA_CANARY_PERMISSIONS_BOUNDARY }}
+        run: npx serverless deploy --stage "$CANARY_STAGE"
+
+      - name: Invoke and assert the capability payload
+        env:
+          LD_AI_CANARY_VERSIONS: ${{ steps.versions.outputs.resolved }}
+        run: |
+          set -euo pipefail
+          aws lambda invoke \
+            --function-name "ld-ai-canary-$CANARY_STAGE" \
+            --cli-binary-format raw-in-base64-out \
+            --payload '{}' \
+            response.json
+          cat response.json
+          node integration-tests/lambda-commonjs/canary.mjs assert --response=response.json
+
+      # A failed build stops before the credentials step. Then no AWS resource exists,
+      # and the cleanup has no credentials to delete one.
+      - name: Remove the ephemeral Lambda
+        if: always() && steps.aws-credentials.outcome == 'success'
+        run: |
+          set -uo pipefail
+          status=0
+          (cd integration-tests/lambda-commonjs/.canary/app && npx serverless remove --stage "$CANARY_STAGE") || status=1
+          log_group="/aws/lambda/ld-ai-canary-$CANARY_STAGE"
+          if ! delete_output=$(aws logs delete-log-group --log-group-name "$log_group" 2>&1); then
+            # Stack removal already deletes the log group, so a missing group is a success.
+            if ! printf '%s' "$delete_output" | grep -q 'ResourceNotFoundException'; then
+              echo "$delete_output"
+              status=1
+            fi
+          fi
+          if [ "$status" -ne 0 ]; then
+            echo "::error::Canary cleanup failed. Look for a leftover ld-ai-canary-$CANARY_STAGE stack or $log_group log group."
+          fi
+          exit "$status"

.gitignore

@@ -27,5 +27,6 @@ output/
 spec.md
 tmp/
 integration-tests/**/.staging/
+integration-tests/**/.canary/
 
 .claude/

biome.json

@@ -48,7 +48,8 @@
       "!**/node_modules",
       "!**/*.d.ts",
       "!**/output",
-      "!**/.staging"
+      "!**/.staging",
+      "!**/.canary"
     ]
   },
   "overrides": [

integration-tests/lambda-commonjs/README.md

@@ -0,0 +1,140 @@
+# Published-artifact Lambda release canary
+
+Owner: the LaunchDarkly AI SDK team. The team maintains `launchdarkly/js-ai-sdk`.
+
+## Plain version
+
+We publish packages to npm. A customer then installs them and runs them on AWS Lambda.
+
+This test does the same thing after each release. It installs the published packages.
+It runs them in a real AWS Lambda function. It then deletes that function.
+
+If the test fails, the packages are broken for customers. We learn this in minutes, not weeks.
+
+The test only loads the packages. It uses no SDK key. It evaluates no flag. It calls no model.
+So the test costs almost nothing.
+
+## What it means for the code (junior eng)
+
+A canary is a small test that runs against production artifacts after a release.
+This canary is the second half of the work in
+[AIC-3370](https://launchdarkly.atlassian.net/browse/AIC-3370).
+
+The suite in `integration-tests/commonjs-webpack` packs the workspace with `npm pack`.
+That suite tests the tarball that we plan to publish.
+This canary tests the artifact that npm actually serves.
+
+The canary installs exact published versions. It never uses a workspace link.
+It never uses a dist-tag such as `latest`.
+
+The fixture bundles the app with Serverless Framework 3 and Webpack 5.
+It also uses `webpack-node-externals` 3 with no allowlist.
+So Node resolves the LaunchDarkly packages at runtime.
+That path is the exact path that failed in AIC-3370.
+
+The workflow then deploys one function. It invokes the function one time.
+It checks the JSON answer. It then deletes the function and the log group.
+
+### The driver
+
+`canary.mjs` holds all of the logic. It takes one exact version for each package.
+You pass the versions as flags. You can also set
+`LD_AI_CANARY_VERSIONS='ai-server=0.4.0,ai-node=0.3.0,openai-messages=0.3.0,ai-otel=0.2.0'`.
+The driver rejects a version that is not an exact semver.
+
+The list includes `@launchdarkly/ai-server`, the core package.
+`@launchdarkly/ai-node` depends on the core package with `*`.
+So npm picks any visible core version when the canary does not pin it.
+The canary installs the core version directly, and a core-only release also runs the canary.
+
+| Command | What it does |
+| --- | --- |
+| `wait` | Calls `npm view` until npm serves all four versions. A new publish is not readable at once. |
+| `build` | Copies `fixture/` into `.canary/app`. Installs the exact versions with `--save-exact`. Checks the installed tree. Runs `serverless package` when you pass `--package`. |
+| `assert` | Reads the invoke payload. Checks `ok`, a `v22.x` runtime, the capability names, and the loaded versions. |
+
+The function name is `ld-ai-canary-<stage>`. The workflow gives each run a unique stage.
+
+### Local dry run
+
+This run needs no AWS credentials.
+
+```bash
+cd integration-tests/lambda-commonjs
+node canary.mjs build --ai-server=0.3.0 --ai-node=0.2.0 --openai-messages=0.2.0 --ai-otel=0.1.1 --package --stage=local
+unzip -l .canary/app/.serverless/ld-ai-canary.zip | grep '@launchdarkly'
+```
+
+The config schema of Serverless 3 does not know the `nodejs22.x` runtime.
+So the package step prints an `Invalid configuration encountered` warning for `provider.runtime`.
+The fixture sets `configValidationMode: warn`, and the message stays a warning.
+CloudFormation accepts the runtime. The function runs on Node 22.
+The canary reads `process.version` and fails on any other runtime.
+
+`yarn test:integration` runs `canary.test.ts`.
+That file tests the assert logic and the exact-version guard.
+It uses no network and no AWS account.
+
+### AWS prerequisite
+
+The workflow job uses GitHub OIDC only.
+OIDC lets the job get short-lived AWS credentials from a role.
+This repository holds no long-lived AWS keys. Please do not add any.
+
+The job stays skipped until these repository variables exist.
+
+| Variable | Purpose |
+| --- | --- |
+| `AWS_LAMBDA_CANARY_ROLE_ARN` | The role that the job assumes. The job is skipped while this variable is empty. |
+| `AWS_LAMBDA_CANARY_PERMISSIONS_BOUNDARY` | The boundary policy ARN. It must match the ARN in the role's `iam:PermissionsBoundary` condition. The job is skipped while this variable is empty. |
+| `AWS_LAMBDA_CANARY_REGION` | Optional. The default is `us-east-1`. |
+
+The trust policy of the role must allow `token.actions.githubusercontent.com`.
+Limit `sub` to `repo:launchdarkly/js-ai-sdk:ref:refs/heads/main`.
+The release push runs from `main`, and you start a canary dispatch from `main` too.
+A wider `sub` lets a workflow on any branch assume the role.
+Set `aud` to `https://github.com/launchdarkly`.
+
+The role needs these permissions. Scope each one to the `ld-ai-canary-*` resources.
+
+- `cloudformation:CreateStack`, `UpdateStack`, `DeleteStack`, `DescribeStacks`, `DescribeStackEvents`,
+  `DescribeStackResource(s)`, `ValidateTemplate`, `GetTemplate`, `ListStackResources`
+- `s3:CreateBucket`, `DeleteBucket`, `ListBucket`, `GetObject`, `PutObject`, `DeleteObject`,
+  `GetBucketLocation`, `GetBucketPolicy`, `PutBucketPolicy`, `PutEncryptionConfiguration` on the
+  Serverless deployment bucket
+- `lambda:CreateFunction`, `DeleteFunction`, `GetFunction`, `GetFunctionConfiguration`,
+  `UpdateFunctionCode`, `UpdateFunctionConfiguration`, `InvokeFunction`, `ListVersionsByFunction`,
+  `TagResource`
+- `logs:CreateLogGroup`, `DeleteLogGroup`, `DescribeLogGroups`, `PutRetentionPolicy`
+- `iam:CreateRole`, `DeleteRole`, `GetRole`, `PassRole`, `PutRolePolicy`, `DeleteRolePolicy`
+  for the execution role of the function
+
+The generated stack puts inline `Policies` on the execution role only.
+So the role needs no `iam:AttachRolePolicy` and no `iam:DetachRolePolicy`.
+
+The `iam:CreateRole` and `iam:PutRolePolicy` actions need an `iam:PermissionsBoundary` condition.
+That condition stops the role from granting itself more rights than the boundary allows.
+The workflow passes `vars.AWS_LAMBDA_CANARY_PERMISSIONS_BOUNDARY` to the fixture.
+The fixture reads it as `LD_AI_CANARY_PERMISSIONS_BOUNDARY`.
+Set that repository variable to the same boundary policy ARN that the condition names.
+
+### When the canary fails
+
+A failure means that a published package does not load from a CommonJS Lambda bundle.
+That is the AIC-3370 bug again.
+
+Run the local dry run above with the published versions.
+Then invoke the generated bundle directly.
+
+The `serverless-webpack` plugin deletes `.webpack/` after it writes the zip.
+So you unzip the package first, and then you load `handler.js` from there.
+
+```bash
+cd integration-tests/lambda-commonjs/.canary/app
+unzip -o .serverless/ld-ai-canary.zip -d .unzipped
+node -e "require('./.unzipped/handler.js').handler().then(console.log, console.error)"
+```
+
+The cleanup step always runs. The step prints an error when it cannot delete the stack.
+Then you delete the `ld-ai-canary-<stage>` CloudFormation stack by hand.
+You also delete the `/aws/lambda/ld-ai-canary-<stage>` log group by hand.