From 4a1ac0e508b6a4b081ab881d50a24045f33f74de Mon Sep 17 00:00:00 2001 From: riita10069 Date: Fri, 21 Aug 2026 16:20:39 +0900 Subject: [PATCH 01/15] feat: add BEVFormer occupancy runtime --- .dockerignore | 9 + Platform/HowToUseFlyte.md | 107 +- Platform/buildspec-dataprep.yml | 20 +- .../buildspec-launch-distributed-canary.yml | 165 + .../buildspec-launch-kitscenes-benchmark.yml | 198 + .../buildspec-launch-l2d-reactive-mini.yml | 196 + .../buildspec-launch-nuplan-acquisition.yml | 191 + Platform/buildspec-publish-occupancy.yml | 199 + Platform/buildspec-register-distributed.yml | 49 + Platform/buildspec-register.yml | 40 +- Platform/buildspec-training.yml | 32 + Platform/buildspec.yml | 13 + Platform/docker/bevformer-v2/Dockerfile | 133 + Platform/docker/data-prep/Dockerfile | 18 + Platform/docker/training/Dockerfile | 15 + Platform/helm-values/flyte-core-eks.yaml | 20 +- Platform/infra/main.tf | 14 +- Platform/infra/modules/codebuild/main.tf | 55 + Platform/infra/modules/ecr/main.tf | 2 +- Platform/infra/modules/flyte/main.tf | 29 + Platform/infra/modules/kuberay/main.tf | 22 + Platform/infra/modules/kueue/main.tf | 4 + Platform/infra/modules/storage/main.tf | 8 +- Platform/infra/post-apply.sh | 9 +- .../karpenter-nodepools/gpu-nodeclass.yaml | 25 + .../k8s/karpenter-nodepools/gpu-nodepool.yaml | 12 +- Platform/k8s/kueue-config/kueue-objects.yaml | 73 +- .../k8s/rayjob-templates/ddp-smoke-4.yaml | 126 + Platform/pipelines/bevformer_v2_occupancy.py | 347 ++ Platform/pipelines/bevformer_v2_runtime.py | 527 +++ Platform/pipelines/distributed_training.py | 705 ++++ Platform/pipelines/nuplan_acquisition.py | 743 ++++ Platform/pipelines/occupancy_store.py | 339 ++ Platform/pipelines/semantic_occupancy.py | 412 ++ .../trajectory_visualization_tasks.py | 2 +- Platform/pipelines/workflows.py | 3721 ++++++++++++++++- 36 files changed, 8366 insertions(+), 214 deletions(-) create mode 100644 .dockerignore create mode 100644 Platform/buildspec-launch-distributed-canary.yml create mode 100644 Platform/buildspec-launch-kitscenes-benchmark.yml create mode 100644 Platform/buildspec-launch-l2d-reactive-mini.yml create mode 100644 Platform/buildspec-launch-nuplan-acquisition.yml create mode 100644 Platform/buildspec-publish-occupancy.yml create mode 100644 Platform/buildspec-register-distributed.yml create mode 100644 Platform/buildspec-training.yml create mode 100644 Platform/docker/bevformer-v2/Dockerfile create mode 100644 Platform/infra/modules/kuberay/main.tf create mode 100644 Platform/k8s/karpenter-nodepools/gpu-nodeclass.yaml create mode 100644 Platform/k8s/rayjob-templates/ddp-smoke-4.yaml create mode 100644 Platform/pipelines/bevformer_v2_occupancy.py create mode 100644 Platform/pipelines/bevformer_v2_runtime.py create mode 100644 Platform/pipelines/distributed_training.py create mode 100644 Platform/pipelines/nuplan_acquisition.py create mode 100644 Platform/pipelines/occupancy_store.py create mode 100644 Platform/pipelines/semantic_occupancy.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..cb24f91cf --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +.git +**/.next +**/.pytest_cache +**/__pycache__ +**/node_modules +**/test-results +**/*.ckpt +**/*.pt +**/*.pth diff --git a/Platform/HowToUseFlyte.md b/Platform/HowToUseFlyte.md index e1f747273..937e8b125 100644 --- a/Platform/HowToUseFlyte.md +++ b/Platform/HowToUseFlyte.md @@ -232,10 +232,10 @@ aws codebuild start-build \ ``` The commit-derived tag keeps active workflows that still reference `latest` -unchanged. Registration and launch resolve the selected `training`, `eval`, -`offline-rl`, and `data-prep` tags to ECR digests. The launcher also recomputes -the preprocessing and inference source digests inside the source bundle; Flyte -tasks reject any mismatch at runtime. +unchanged. Registration resolves the selected `training`, `eval`, `offline-rl`, +`data-prep`, and `bevformer-v2` tags to ECR digests. The launcher also +recomputes the preprocessing and inference source digests inside the source +bundle; Flyte tasks reject any mismatch at runtime. ### Launch the one-episode smoke @@ -373,6 +373,104 @@ inputs. It is cached by immutable input URI and report schema. --- +## Use case I — "I want to publish the KITScenes occupancy models" + +**You are**: a platform operator publishing the pinned native AutoE2E +segmentation and official BEVFormer V2 detection footprints for the Occupancy +Dashboard. + +Use only the VPC-local CodeBuild project +`auto-e2e-platform-occupancy-publish`. Its launcher submits these two workflows: + +- `wf_precompute_semantic_occupancy` +- `wf_precompute_bevformer_v2_occupancy` + +The production recipe is fixed to these inputs: + +| Input | Required identity | +|------|-------------------| +| AutoE2E checkpoint | `e41978b037986bc874ec9eec0aebf732c096ae5bfa64cee9c5fb3a4168e87a01` | +| BEVFormer V2 weight | `5585bc4d3ff8b396928cb92d91f773a2c57a81258f83cab0c668ebb2eb9d3307` | +| KITScenes v3.3 manifest | `31faf5d2ceef17522f1c79e6ab31558a2aceb6f75043a130d1f6b7da3ce6211d` | +| BEVFormer score threshold | `0.2` | + +### Stage the official weight once + +Download `epoch_24.pth` from the official BEVFormer V2 release linked in the +model provenance. Verify it locally, then create the canonical checkpoint +object without replacing an existing object: + +```bash +export AWS_PROFILE=autowarefoundation +export AWS_REGION=us-west-2 +ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) +CHECKPOINTS_BUCKET="auto-e2e-platform-checkpoints-${ACCOUNT_ID}" +BEVFORMER_SHA=5585bc4d3ff8b396928cb92d91f773a2c57a81258f83cab0c668ebb2eb9d3307 +BEVFORMER_KEY="external/bevformer-v2/${BEVFORMER_SHA}/epoch_24.pth" + +test "$(shasum -a 256 /path/to/epoch_24.pth | cut -d' ' -f1)" = \ + "${BEVFORMER_SHA}" +aws s3api put-object \ + --bucket "${CHECKPOINTS_BUCKET}" \ + --key "${BEVFORMER_KEY}" \ + --body /path/to/epoch_24.pth \ + --if-none-match '*' \ + --metadata "sha256=${BEVFORMER_SHA}" +``` + +If the conditional upload reports that the object exists, use `head-object` and +verify it independently. Do not overwrite the canonical key. + +### Build, register, and launch + +Apply the tested Terraform first so the ECR repository and dedicated CodeBuild +project exist. Then archive the exact tested commit and build all five runtime +images as in Use case G: + +```bash +REPOSITORY_REVISION=$(git rev-parse HEAD) +IMAGE_TAG="occupancy-${REPOSITORY_REVISION:0:12}" +CACHE_BUCKET="auto-e2e-platform-codebuild-cache-${ACCOUNT_ID}" +PUBLICATION_TIMESTAMP=2026-08-20T00:00:00Z + +git archive --format=zip --output=/tmp/auto-e2e-source.zip HEAD +aws s3 cp /tmp/auto-e2e-source.zip "s3://${CACHE_BUCKET}/source.zip" + +aws codebuild start-build \ + --project-name auto-e2e-platform-build-images \ + --environment-variables-override \ + "name=IMAGE_TAG,value=${IMAGE_TAG},type=PLAINTEXT" + +# Wait for the image build to reach SUCCEEDED before continuing. +aws codebuild start-build \ + --project-name auto-e2e-platform-flyte-register \ + --environment-variables-override \ + "name=IMAGE_TAG,value=${IMAGE_TAG},type=PLAINTEXT" + +# Wait for registration to reach SUCCEEDED before continuing. +aws codebuild start-build \ + --project-name auto-e2e-platform-occupancy-publish \ + --environment-variables-override \ + "name=IMAGE_TAG,value=${IMAGE_TAG},type=PLAINTEXT" \ + "name=PUBLICATION_TIMESTAMP,value=${PUBLICATION_TIMESTAMP},type=PLAINTEXT" \ + "name=REPOSITORY_REVISION,value=${REPOSITORY_REVISION},type=PLAINTEXT" +``` + +Wait for each CodeBuild invocation to succeed before starting the next one. +The publication launcher downloads and verifies both checkpoints and the +published manifest before resolving all five ECR tags to immutable digests. + +A successful publication CodeBuild run means both remote executions were +submitted. In Flyte Console, wait for both occupancy workflows to succeed. +Their outputs report the immutable manifest key, manifest SHA-256, checkpoint +SHA-256, shard count, and sample count. The Dashboard advertises a model only +after its schema-v2 manifest is present. + +Keep `PUBLICATION_TIMESTAMP` unchanged when retrying the same recipe. Changing +it would produce conflicting bytes at the recipe-stable manifest key. + +--- + ## Reading the DAG of `wf_full_pipeline` ``` @@ -424,6 +522,7 @@ n1 data_processing(L2D) n3 data_processing(NVIDIA) ← run in parallel | Publish existing shards only | `wf_publish_dataset_snapshot` | `shards`, `published_dataset`, `dataset_version` | | Precompute an already identified snapshot | `wf_precompute_overlays` | `shards`, model version, dataset manifest digest | | Export an overlay shard as MP4 | `wf_export_trajectory_report` | matching immutable `shard`, `overlay`, dataset-manifest, and overlay-manifest URIs | +| Publish KITScenes occupancy models | `wf_precompute_semantic_occupancy`, `wf_precompute_bevformer_v2_occupancy` | ops-only CodeBuild launch, pinned checkpoints and manifest | | Train IL from existing shards | `wf_train_il` | `shards` list, `dataset` | | Refine with Offline RL | `wf_train_offline_rl` | `pretrained`, `il_metadata`, `shards` | | See metrics | (MLflow, not Flyte) | experiment `imitation-learning` / `offline-rl` | diff --git a/Platform/buildspec-dataprep.yml b/Platform/buildspec-dataprep.yml index cc14179fa..580429638 100644 --- a/Platform/buildspec-dataprep.yml +++ b/Platform/buildspec-dataprep.yml @@ -3,18 +3,34 @@ version: 0.2 env: variables: AWS_DEFAULT_REGION: us-west-2 + IMAGE_TAG: "" phases: pre_build: commands: + - test -n "${IMAGE_TAG}" - ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) - ECR_URL="${ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com" - aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $ECR_URL + - docker pull "${ECR_URL}/auto-e2e/data-prep:latest" || true build: commands: - echo "=== Building data-prep image ===" - - docker build -t ${ECR_URL}/auto-e2e/data-prep:latest -f Platform/docker/data-prep/Dockerfile . - - docker push ${ECR_URL}/auto-e2e/data-prep:latest + - > + DOCKER_BUILDKIT=1 docker build + --cache-from "${ECR_URL}/auto-e2e/data-prep:latest" + --build-arg BUILDKIT_INLINE_CACHE=1 + -t "${ECR_URL}/auto-e2e/data-prep:${IMAGE_TAG}" + -f Platform/docker/data-prep/Dockerfile . + - docker push "${ECR_URL}/auto-e2e/data-prep:${IMAGE_TAG}" + post_build: + commands: + - > + aws ecr describe-images + --repository-name auto-e2e/data-prep + --image-ids imageTag="${IMAGE_TAG}" + --query 'imageDetails[0].{digest:imageDigest,pushed:imagePushedAt,size:imageSizeInBytes}' + --output json cache: paths: diff --git a/Platform/buildspec-launch-distributed-canary.yml b/Platform/buildspec-launch-distributed-canary.yml new file mode 100644 index 000000000..3945b659b --- /dev/null +++ b/Platform/buildspec-launch-distributed-canary.yml @@ -0,0 +1,165 @@ +version: 0.2 + +env: + variables: + AWS_DEFAULT_REGION: us-west-2 + FLYTE_PROJECT: auto-e2e + FLYTE_DOMAIN: development + WORKFLOW_NAME: >- + Platform.pipelines.distributed_training.wf_reactive_multistage_ray_2_canary + FLYTE_VERSION: "" + EXECUTION_NAME: "" + WAIT_TIMEOUT_SECONDS: "3300" + +phases: + install: + commands: + - pip install -q flytekit==1.16.24 + pre_build: + commands: + - test -n "${FLYTE_VERSION}" + - test -n "${EXECUTION_NAME}" + - printf '%s' "${EXECUTION_NAME}" | grep -Eq '^[a-z0-9]([-a-z0-9]*[a-z0-9])?$' + - test "${#EXECUTION_NAME}" -le 63 + - printf '%s' "${WAIT_TIMEOUT_SECONDS}" | grep -Eq '^[1-9][0-9]*$' + - | + cat > /tmp/flyte.yaml << EOF + admin: + endpoint: dns:///k8s-flyte-flyteadm-be92e7737c-cf0d1a0f3df10f77.elb.us-west-2.amazonaws.com:81 + insecure: true + EOF + build: + commands: + - | + python3 - <<'PY' + import os + import time + + import grpc + from flytekit.configuration import Config + from flytekit.exceptions.user import ( + FlyteEntityAlreadyExistsException, + FlyteEntityNotExistException, + ) + from flytekit.models.core.execution import WorkflowExecutionPhase + from flytekit.remote import FlyteRemote + + project = os.environ["FLYTE_PROJECT"] + domain = os.environ["FLYTE_DOMAIN"] + workflow_name = os.environ["WORKFLOW_NAME"] + version = os.environ["FLYTE_VERSION"] + execution_name = os.environ["EXECUTION_NAME"] + timeout_seconds = int(os.environ["WAIT_TIMEOUT_SECONDS"]) + + remote = FlyteRemote( + config=Config.auto(config_file="/tmp/flyte.yaml"), + default_project=project, + default_domain=domain, + ) + launch_plan = remote.fetch_launch_plan( + project=project, + domain=domain, + name=workflow_name, + version=version, + ) + + def grpc_code(error): + visited = set() + candidate = error + while candidate is not None and id(candidate) not in visited: + visited.add(id(candidate)) + if isinstance(candidate, grpc.RpcError): + return candidate.code() + candidate = ( + candidate.__cause__ + if candidate.__cause__ is not None + else candidate.__context__ + ) + return None + + transient_codes = { + grpc.StatusCode.DEADLINE_EXCEEDED, + grpc.StatusCode.INTERNAL, + grpc.StatusCode.RESOURCE_EXHAUSTED, + grpc.StatusCode.UNAVAILABLE, + grpc.StatusCode.UNKNOWN, + } + launch_deadline = time.monotonic() + 300 + execution = None + while execution is None: + try: + execution = remote.fetch_execution( + project=project, + domain=domain, + name=execution_name, + ) + break + except FlyteEntityNotExistException: + pass + except Exception as fetch_error: + fetch_code = grpc_code(fetch_error) + if fetch_code not in transient_codes: + raise + try: + execution = remote.execute( + launch_plan, + inputs={}, + execution_name=execution_name, + wait=False, + ) + except FlyteEntityAlreadyExistsException: + pass + except Exception as create_error: + create_code = grpc_code(create_error) + if create_code not in transient_codes: + raise + if execution is None: + if time.monotonic() >= launch_deadline: + raise TimeoutError( + "timed out fetching or creating Flyte execution" + ) + time.sleep(5) + + print(f"FLYTE_EXECUTION_ID={execution.id.name}", flush=True) + wait_deadline = time.monotonic() + timeout_seconds + terminal_phases = { + WorkflowExecutionPhase.ABORTED, + WorkflowExecutionPhase.FAILED, + WorkflowExecutionPhase.SUCCEEDED, + WorkflowExecutionPhase.TIMED_OUT, + } + while True: + try: + execution = remote.sync_execution( + execution, + sync_nodes=True, + ) + except Exception as sync_error: + if grpc_code(sync_error) not in transient_codes: + raise + print( + "FLYTE_ADMIN_TRANSIENT_RETRY=" + f"{grpc_code(sync_error).name}", + flush=True, + ) + else: + if execution.closure.phase in terminal_phases: + break + if time.monotonic() >= wait_deadline: + raise TimeoutError( + f"timed out waiting for Flyte execution {execution_name}" + ) + time.sleep(10) + + phase = execution.closure.phase + phase_name = WorkflowExecutionPhase.enum_to_string(phase) + print(f"FLYTE_EXECUTION_PHASE={phase_name}", flush=True) + if phase != WorkflowExecutionPhase.SUCCEEDED: + error = execution.error + if error is not None: + print(f"FLYTE_EXECUTION_ERROR={error}", flush=True) + raise SystemExit( + f"Flyte execution {execution.id.name} ended in {phase_name}" + ) + print(f"FLYTE_EXECUTION_OUTPUTS={execution.outputs}", flush=True) + PY diff --git a/Platform/buildspec-launch-kitscenes-benchmark.yml b/Platform/buildspec-launch-kitscenes-benchmark.yml new file mode 100644 index 000000000..495f30bda --- /dev/null +++ b/Platform/buildspec-launch-kitscenes-benchmark.yml @@ -0,0 +1,198 @@ +version: 0.2 + +env: + variables: + AWS_DEFAULT_REGION: us-west-2 + FLYTE_PROJECT: auto-e2e + FLYTE_DOMAIN: development + AUDIT_WORKFLOW_NAME: >- + Platform.pipelines.workflows.wf_audit_kitscenes_benchmark_inventory + PREPARE_WORKFLOW_NAME: >- + Platform.pipelines.workflows.wf_prepare_kitscenes_paper_approximation + MODE: audit + FLYTE_VERSION: "" + EXECUTION_NAME: "" + VAL_SCENE_LIMIT: "0" + OVERLAP_SCENE_LIMIT: "0" + INGEST_CONCURRENCY: "20" + PACK_CONCURRENCY: "20" + RELEASE_ID: autoe2e-paper-approx-v1 + WAIT_TIMEOUT_SECONDS: "21600" + +phases: + install: + commands: + - pip install -q flytekit==1.16.24 + pre_build: + commands: + - test -n "${FLYTE_VERSION}" + - test -n "${EXECUTION_NAME}" + - printf '%s' "${MODE}" | grep -Eq '^(audit|prepare)$' + - printf '%s' "${EXECUTION_NAME}" | grep -Eq '^[a-z0-9]([-a-z0-9]*[a-z0-9])?$' + - test "${#EXECUTION_NAME}" -le 63 + - printf '%s' "${VAL_SCENE_LIMIT}" | grep -Eq '^[0-9]+$' + - printf '%s' "${OVERLAP_SCENE_LIMIT}" | grep -Eq '^[0-9]+$' + - printf '%s' "${INGEST_CONCURRENCY}" | grep -Eq '^[1-9][0-9]*$' + - printf '%s' "${PACK_CONCURRENCY}" | grep -Eq '^[1-9][0-9]*$' + - printf '%s' "${WAIT_TIMEOUT_SECONDS}" | grep -Eq '^[1-9][0-9]*$' + - | + cat > /tmp/flyte.yaml << EOF + admin: + endpoint: dns:///k8s-flyte-flyteadm-be92e7737c-cf0d1a0f3df10f77.elb.us-west-2.amazonaws.com:81 + insecure: true + EOF + build: + commands: + - | + python3 - <<'PY' + import os + import time + + import grpc + from flytekit.configuration import Config + from flytekit.exceptions.user import ( + FlyteEntityAlreadyExistsException, + FlyteEntityNotExistException, + ) + from flytekit.models.core.execution import WorkflowExecutionPhase + from flytekit.remote import FlyteRemote + + project = os.environ["FLYTE_PROJECT"] + domain = os.environ["FLYTE_DOMAIN"] + mode = os.environ["MODE"] + workflow_name = ( + os.environ["AUDIT_WORKFLOW_NAME"] + if mode == "audit" + else os.environ["PREPARE_WORKFLOW_NAME"] + ) + version = os.environ["FLYTE_VERSION"] + execution_name = os.environ["EXECUTION_NAME"] + timeout_seconds = int(os.environ["WAIT_TIMEOUT_SECONDS"]) + inputs = {} + if mode == "prepare": + inputs = { + "val_scene_limit": int(os.environ["VAL_SCENE_LIMIT"]), + "overlap_scene_limit": int( + os.environ["OVERLAP_SCENE_LIMIT"] + ), + "ingest_concurrency": int( + os.environ["INGEST_CONCURRENCY"] + ), + "pack_concurrency": int( + os.environ["PACK_CONCURRENCY"] + ), + "release_id": os.environ["RELEASE_ID"], + } + + remote = FlyteRemote( + config=Config.auto(config_file="/tmp/flyte.yaml"), + default_project=project, + default_domain=domain, + ) + launch_plan = remote.fetch_launch_plan( + project=project, + domain=domain, + name=workflow_name, + version=version, + ) + + def grpc_code(error): + visited = set() + candidate = error + while candidate is not None and id(candidate) not in visited: + visited.add(id(candidate)) + if isinstance(candidate, grpc.RpcError): + return candidate.code() + candidate = ( + candidate.__cause__ + if candidate.__cause__ is not None + else candidate.__context__ + ) + return None + + transient_codes = { + grpc.StatusCode.DEADLINE_EXCEEDED, + grpc.StatusCode.INTERNAL, + grpc.StatusCode.RESOURCE_EXHAUSTED, + grpc.StatusCode.UNAVAILABLE, + grpc.StatusCode.UNKNOWN, + } + launch_deadline = time.monotonic() + 300 + execution = None + while execution is None: + try: + execution = remote.fetch_execution( + project=project, + domain=domain, + name=execution_name, + ) + break + except FlyteEntityNotExistException: + pass + except Exception as fetch_error: + if grpc_code(fetch_error) not in transient_codes: + raise + try: + execution = remote.execute( + launch_plan, + inputs=inputs, + execution_name=execution_name, + wait=False, + ) + except FlyteEntityAlreadyExistsException: + pass + except Exception as create_error: + if grpc_code(create_error) not in transient_codes: + raise + if execution is None: + if time.monotonic() >= launch_deadline: + raise TimeoutError( + "timed out fetching or creating Flyte execution" + ) + time.sleep(5) + + print(f"FLYTE_EXECUTION_ID={execution.id.name}", flush=True) + wait_deadline = time.monotonic() + timeout_seconds + terminal_phases = { + WorkflowExecutionPhase.ABORTED, + WorkflowExecutionPhase.FAILED, + WorkflowExecutionPhase.SUCCEEDED, + WorkflowExecutionPhase.TIMED_OUT, + } + while True: + try: + execution = remote.sync_execution( + execution, + sync_nodes=True, + ) + except Exception as sync_error: + code = grpc_code(sync_error) + if code not in transient_codes: + raise + print( + f"FLYTE_ADMIN_TRANSIENT_RETRY={code.name}", + flush=True, + ) + else: + if execution.closure.phase in terminal_phases: + break + if time.monotonic() >= wait_deadline: + raise TimeoutError( + f"timed out waiting for Flyte execution {execution_name}" + ) + time.sleep(10) + + phase = execution.closure.phase + phase_name = WorkflowExecutionPhase.enum_to_string(phase) + print(f"FLYTE_EXECUTION_PHASE={phase_name}", flush=True) + if phase != WorkflowExecutionPhase.SUCCEEDED: + if execution.error is not None: + print( + f"FLYTE_EXECUTION_ERROR={execution.error}", + flush=True, + ) + raise SystemExit( + f"Flyte execution {execution.id.name} ended in {phase_name}" + ) + print(f"FLYTE_EXECUTION_OUTPUTS={execution.outputs}", flush=True) + PY diff --git a/Platform/buildspec-launch-l2d-reactive-mini.yml b/Platform/buildspec-launch-l2d-reactive-mini.yml new file mode 100644 index 000000000..db9548ea8 --- /dev/null +++ b/Platform/buildspec-launch-l2d-reactive-mini.yml @@ -0,0 +1,196 @@ +version: 0.2 + +env: + variables: + AWS_DEFAULT_REGION: us-west-2 + FLYTE_PROJECT: auto-e2e + FLYTE_DOMAIN: development + WORKFLOW_NAME: >- + Platform.pipelines.workflows.wf_prepare_l2d_reactive_dataset + FLYTE_VERSION: "" + EXECUTION_NAME: "" + SOURCE_PBF_URI: "" + SOURCE_REVISION: "" + SOURCE_DATE: "" + START_EP: "185" + END_EP: "188" + PARTITION_SIZE: "1" + MAX_PARTITIONS: "3" + INGEST_CONCURRENCY: "3" + PACK_CONCURRENCY: "3" + WAIT_TIMEOUT_SECONDS: "7200" + +phases: + install: + commands: + - pip install -q flytekit==1.16.24 + pre_build: + commands: + - test -n "${FLYTE_VERSION}" + - test -n "${EXECUTION_NAME}" + - test -n "${SOURCE_PBF_URI}" + - test -n "${SOURCE_REVISION}" + - test -n "${SOURCE_DATE}" + - printf '%s' "${SOURCE_PBF_URI}" | grep -Eq '^s3://' + - printf '%s' "${EXECUTION_NAME}" | grep -Eq '^[a-z0-9]([-a-z0-9]*[a-z0-9])?$' + - test "${#EXECUTION_NAME}" -le 63 + - printf '%s' "${START_EP}" | grep -Eq '^[0-9]+$' + - printf '%s' "${END_EP}" | grep -Eq '^[0-9]+$' + - test "${END_EP}" -gt "${START_EP}" + - printf '%s' "${WAIT_TIMEOUT_SECONDS}" | grep -Eq '^[1-9][0-9]*$' + - | + cat > /tmp/flyte.yaml << EOF + admin: + endpoint: dns:///k8s-flyte-flyteadm-be92e7737c-cf0d1a0f3df10f77.elb.us-west-2.amazonaws.com:81 + insecure: true + EOF + build: + commands: + - | + python3 - <<'PY' + import os + import time + + import grpc + from flytekit.configuration import Config + from flytekit.exceptions.user import ( + FlyteEntityAlreadyExistsException, + FlyteEntityNotExistException, + ) + from flytekit.models.core.execution import WorkflowExecutionPhase + from flytekit.remote import FlyteRemote + from flytekit.types.file import FlyteFile + + project = os.environ["FLYTE_PROJECT"] + domain = os.environ["FLYTE_DOMAIN"] + workflow_name = os.environ["WORKFLOW_NAME"] + version = os.environ["FLYTE_VERSION"] + execution_name = os.environ["EXECUTION_NAME"] + timeout_seconds = int(os.environ["WAIT_TIMEOUT_SECONDS"]) + inputs = { + "source_pbf": FlyteFile(os.environ["SOURCE_PBF_URI"]), + "source_revision": os.environ["SOURCE_REVISION"], + "source_date": os.environ["SOURCE_DATE"], + "episodes": 0, + "start_ep": int(os.environ["START_EP"]), + "end_ep": int(os.environ["END_EP"]), + "partition_size": int(os.environ["PARTITION_SIZE"]), + "max_partitions": int(os.environ["MAX_PARTITIONS"]), + "ingest_concurrency": int( + os.environ["INGEST_CONCURRENCY"] + ), + "pack_concurrency": int(os.environ["PACK_CONCURRENCY"]), + } + + remote = FlyteRemote( + config=Config.auto(config_file="/tmp/flyte.yaml"), + default_project=project, + default_domain=domain, + ) + launch_plan = remote.fetch_launch_plan( + project=project, + domain=domain, + name=workflow_name, + version=version, + ) + + def grpc_code(error): + visited = set() + candidate = error + while candidate is not None and id(candidate) not in visited: + visited.add(id(candidate)) + if isinstance(candidate, grpc.RpcError): + return candidate.code() + candidate = ( + candidate.__cause__ + if candidate.__cause__ is not None + else candidate.__context__ + ) + return None + + transient_codes = { + grpc.StatusCode.DEADLINE_EXCEEDED, + grpc.StatusCode.INTERNAL, + grpc.StatusCode.RESOURCE_EXHAUSTED, + grpc.StatusCode.UNAVAILABLE, + grpc.StatusCode.UNKNOWN, + } + launch_deadline = time.monotonic() + 300 + execution = None + while execution is None: + try: + execution = remote.fetch_execution( + project=project, + domain=domain, + name=execution_name, + ) + break + except FlyteEntityNotExistException: + pass + except Exception as fetch_error: + if grpc_code(fetch_error) not in transient_codes: + raise + try: + execution = remote.execute( + launch_plan, + inputs=inputs, + execution_name=execution_name, + wait=False, + ) + except FlyteEntityAlreadyExistsException: + pass + except Exception as create_error: + if grpc_code(create_error) not in transient_codes: + raise + if execution is None: + if time.monotonic() >= launch_deadline: + raise TimeoutError( + "timed out fetching or creating Flyte execution" + ) + time.sleep(5) + + print(f"FLYTE_EXECUTION_ID={execution.id.name}", flush=True) + wait_deadline = time.monotonic() + timeout_seconds + terminal_phases = { + WorkflowExecutionPhase.ABORTED, + WorkflowExecutionPhase.FAILED, + WorkflowExecutionPhase.SUCCEEDED, + WorkflowExecutionPhase.TIMED_OUT, + } + while True: + try: + execution = remote.sync_execution( + execution, + sync_nodes=True, + ) + except Exception as sync_error: + code = grpc_code(sync_error) + if code not in transient_codes: + raise + print( + f"FLYTE_ADMIN_TRANSIENT_RETRY={code.name}", + flush=True, + ) + else: + if execution.closure.phase in terminal_phases: + break + if time.monotonic() >= wait_deadline: + raise TimeoutError( + f"timed out waiting for Flyte execution {execution_name}" + ) + time.sleep(10) + + phase = execution.closure.phase + phase_name = WorkflowExecutionPhase.enum_to_string(phase) + print(f"FLYTE_EXECUTION_PHASE={phase_name}", flush=True) + if phase != WorkflowExecutionPhase.SUCCEEDED: + if execution.error is not None: + print( + f"FLYTE_EXECUTION_ERROR={execution.error}", + flush=True, + ) + raise SystemExit( + f"Flyte execution {execution.id.name} ended in {phase_name}" + ) + print(f"FLYTE_EXECUTION_OUTPUTS={execution.outputs}", flush=True) + PY diff --git a/Platform/buildspec-launch-nuplan-acquisition.yml b/Platform/buildspec-launch-nuplan-acquisition.yml new file mode 100644 index 000000000..c9e352702 --- /dev/null +++ b/Platform/buildspec-launch-nuplan-acquisition.yml @@ -0,0 +1,191 @@ +version: 0.2 + +env: + variables: + AWS_DEFAULT_REGION: us-west-2 + FLYTE_PROJECT: auto-e2e + FLYTE_DOMAIN: development + WORKFLOW_NAME: >- + Platform.pipelines.workflows.wf_acquire_nuplan_raw_snapshot + FLYTE_VERSION: "" + EXECUTION_NAME: "" + SOURCE_MANIFEST_URI: "" + DATASETS_BUCKET: "" + IMPORT_CONCURRENCY: "4" + WAIT_FOR_COMPLETION: "true" + WAIT_TIMEOUT_SECONDS: "86400" + +phases: + install: + commands: + - pip install -q flytekit==1.16.24 + pre_build: + commands: + - test -n "${FLYTE_VERSION}" + - test -n "${EXECUTION_NAME}" + - test -n "${SOURCE_MANIFEST_URI}" + - test -n "${DATASETS_BUCKET}" + - printf '%s' "${SOURCE_MANIFEST_URI}" | grep -Eq '^s3://' + - printf '%s' "${DATASETS_BUCKET}" | grep -Eq '^[a-z0-9][a-z0-9.-]+$' + - printf '%s' "${EXECUTION_NAME}" | grep -Eq '^[a-z0-9]([-a-z0-9]*[a-z0-9])?$' + - test "${#EXECUTION_NAME}" -le 63 + - printf '%s' "${IMPORT_CONCURRENCY}" | grep -Eq '^[1-9][0-9]*$' + - printf '%s' "${WAIT_FOR_COMPLETION}" | grep -Eq '^(true|false)$' + - printf '%s' "${WAIT_TIMEOUT_SECONDS}" | grep -Eq '^[1-9][0-9]*$' + - | + cat > /tmp/flyte.yaml << EOF + admin: + endpoint: dns:///k8s-flyte-flyteadm-be92e7737c-cf0d1a0f3df10f77.elb.us-west-2.amazonaws.com:81 + insecure: true + EOF + build: + commands: + - | + python3 - <<'PY' + import os + import time + + import grpc + from flytekit.configuration import Config + from flytekit.exceptions.user import ( + FlyteEntityAlreadyExistsException, + FlyteEntityNotExistException, + ) + from flytekit.models.core.execution import WorkflowExecutionPhase + from flytekit.remote import FlyteRemote + from flytekit.types.file import FlyteFile + + project = os.environ["FLYTE_PROJECT"] + domain = os.environ["FLYTE_DOMAIN"] + workflow_name = os.environ["WORKFLOW_NAME"] + version = os.environ["FLYTE_VERSION"] + execution_name = os.environ["EXECUTION_NAME"] + wait_for_completion = ( + os.environ["WAIT_FOR_COMPLETION"] == "true" + ) + timeout_seconds = int(os.environ["WAIT_TIMEOUT_SECONDS"]) + inputs = { + "source_manifest": FlyteFile( + os.environ["SOURCE_MANIFEST_URI"] + ), + "datasets_bucket": os.environ["DATASETS_BUCKET"], + "aws_region": os.environ["AWS_DEFAULT_REGION"], + "concurrency": int(os.environ["IMPORT_CONCURRENCY"]), + } + + remote = FlyteRemote( + config=Config.auto(config_file="/tmp/flyte.yaml"), + default_project=project, + default_domain=domain, + ) + launch_plan = remote.fetch_launch_plan( + project=project, + domain=domain, + name=workflow_name, + version=version, + ) + + def grpc_code(error): + visited = set() + candidate = error + while candidate is not None and id(candidate) not in visited: + visited.add(id(candidate)) + if isinstance(candidate, grpc.RpcError): + return candidate.code() + candidate = ( + candidate.__cause__ + if candidate.__cause__ is not None + else candidate.__context__ + ) + return None + + transient_codes = { + grpc.StatusCode.DEADLINE_EXCEEDED, + grpc.StatusCode.INTERNAL, + grpc.StatusCode.RESOURCE_EXHAUSTED, + grpc.StatusCode.UNAVAILABLE, + grpc.StatusCode.UNKNOWN, + } + launch_deadline = time.monotonic() + 300 + execution = None + while execution is None: + try: + execution = remote.fetch_execution( + project=project, + domain=domain, + name=execution_name, + ) + break + except FlyteEntityNotExistException: + pass + except Exception as fetch_error: + if grpc_code(fetch_error) not in transient_codes: + raise + try: + execution = remote.execute( + launch_plan, + inputs=inputs, + execution_name=execution_name, + wait=False, + ) + except FlyteEntityAlreadyExistsException: + pass + except Exception as create_error: + if grpc_code(create_error) not in transient_codes: + raise + if execution is None: + if time.monotonic() >= launch_deadline: + raise TimeoutError( + "timed out fetching or creating Flyte execution" + ) + time.sleep(5) + + print(f"FLYTE_EXECUTION_ID={execution.id.name}", flush=True) + if not wait_for_completion: + print("FLYTE_EXECUTION_DETACHED=true", flush=True) + raise SystemExit(0) + + wait_deadline = time.monotonic() + timeout_seconds + terminal_phases = { + WorkflowExecutionPhase.ABORTED, + WorkflowExecutionPhase.FAILED, + WorkflowExecutionPhase.SUCCEEDED, + WorkflowExecutionPhase.TIMED_OUT, + } + while True: + try: + execution = remote.sync_execution( + execution, + sync_nodes=True, + ) + except Exception as sync_error: + code = grpc_code(sync_error) + if code not in transient_codes: + raise + print( + f"FLYTE_ADMIN_TRANSIENT_RETRY={code.name}", + flush=True, + ) + else: + if execution.closure.phase in terminal_phases: + break + if time.monotonic() >= wait_deadline: + raise TimeoutError( + f"timed out waiting for Flyte execution {execution_name}" + ) + time.sleep(15) + + phase = execution.closure.phase + phase_name = WorkflowExecutionPhase.enum_to_string(phase) + print(f"FLYTE_EXECUTION_PHASE={phase_name}", flush=True) + if phase != WorkflowExecutionPhase.SUCCEEDED: + if execution.error is not None: + print( + f"FLYTE_EXECUTION_ERROR={execution.error}", + flush=True, + ) + raise SystemExit( + f"Flyte execution {execution.id.name} ended in {phase_name}" + ) + print(f"FLYTE_EXECUTION_OUTPUTS={execution.outputs}", flush=True) + PY diff --git a/Platform/buildspec-publish-occupancy.yml b/Platform/buildspec-publish-occupancy.yml new file mode 100644 index 000000000..8872d5af2 --- /dev/null +++ b/Platform/buildspec-publish-occupancy.yml @@ -0,0 +1,199 @@ +version: 0.2 + +# Publish the pinned native and BEVFormer V2 occupancy sets for KITScenes v3.3. +# +# Required start-build overrides: +# IMAGE_TAG, PUBLICATION_TIMESTAMP, REPOSITORY_REVISION +# Optional: +# BEVFORMER_CHECKPOINT_URI and per-image tags + +env: + shell: bash + variables: + AWS_DEFAULT_REGION: us-west-2 + FLYTE_PROJECT: auto-e2e + FLYTE_DOMAIN: development + IMAGE_TAG: "" + TRAINING_IMAGE_TAG: "" + DATA_PREP_IMAGE_TAG: "" + EVAL_IMAGE_TAG: "" + OFFLINE_RL_IMAGE_TAG: "" + BEVFORMER_V2_IMAGE_TAG: "" + AUTOE2E_CHECKPOINT_URI: "" + BEVFORMER_CHECKPOINT_URI: "" + PUBLISHED_SHARDS_URI: "" + ARTIFACTS_BUCKET: "" + DATASET: kitscenes + DATASET_VERSION: v3.3 + DATASET_MANIFEST_SHA256: 31faf5d2ceef17522f1c79e6ab31558a2aceb6f75043a130d1f6b7da3ce6211d + PUBLICATION_TIMESTAMP: "" + REPOSITORY_REVISION: "" + SCORE_THRESHOLD: "0.2" + +phases: + install: + commands: + - pip install -q flytekit==1.14.9 numpy torch kubernetes --extra-index-url https://download.pytorch.org/whl/cpu + build: + commands: + - ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) + - ECR_URL="${ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com" + - test -n "${IMAGE_TAG}" + - test "${DATASET}" = "kitscenes" + - test "${DATASET_VERSION}" = "v3.3" + - test "${SCORE_THRESHOLD}" = "0.2" + - test "${DATASET_MANIFEST_SHA256}" = "31faf5d2ceef17522f1c79e6ab31558a2aceb6f75043a130d1f6b7da3ce6211d" + - test "${PUBLICATION_TIMESTAMP}" != "" + - test "${REPOSITORY_REVISION}" != "" + - | + [[ "${PUBLICATION_TIMESTAMP}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] + [[ "${REPOSITORY_REVISION}" =~ ^[0-9a-f]{40}$ ]] + - | + if [ -z "${AUTOE2E_CHECKPOINT_URI}" ]; then + AUTOE2E_CHECKPOINT_URI="s3://auto-e2e-platform-checkpoints-${ACCOUNT_ID}/ray-train/nuplan-mini-stage-a-4gpu-9c8a2786-r0-nuplan_full-ray-4/checkpoint_2026-08-19_13-37-37.087949/checkpoint.pt" + fi + if [ -z "${BEVFORMER_CHECKPOINT_URI}" ]; then + BEVFORMER_CHECKPOINT_URI="s3://auto-e2e-platform-checkpoints-${ACCOUNT_ID}/external/bevformer-v2/5585bc4d3ff8b396928cb92d91f773a2c57a81258f83cab0c668ebb2eb9d3307/epoch_24.pth" + fi + if [ -z "${PUBLISHED_SHARDS_URI}" ]; then + PUBLISHED_SHARDS_URI="s3://auto-e2e-platform-datasets-${ACCOUNT_ID}/kitscenes/v3.3/shards" + fi + if [ -z "${ARTIFACTS_BUCKET}" ]; then + ARTIFACTS_BUCKET="auto-e2e-platform-artifacts-${ACCOUNT_ID}" + fi + export AUTOE2E_CHECKPOINT_URI BEVFORMER_CHECKPOINT_URI + export PUBLISHED_SHARDS_URI ARTIFACTS_BUCKET + - | + verify_s3_sha256() { + URI="$1" + EXPECTED="$2" + OUTPUT="$3" + case "${URI}" in + s3://*) ;; + *) + echo "Expected an s3:// URI, got ${URI}" >&2 + return 1 + ;; + esac + aws s3 cp "${URI}" "${OUTPUT}" --only-show-errors + ACTUAL="$(sha256sum "${OUTPUT}" | cut -d' ' -f1)" + if [ "${ACTUAL}" != "${EXPECTED}" ]; then + echo "SHA-256 mismatch for ${URI}: expected ${EXPECTED}, got ${ACTUAL}" >&2 + return 1 + fi + } + verify_s3_sha256 \ + "${AUTOE2E_CHECKPOINT_URI}" \ + "e41978b037986bc874ec9eec0aebf732c096ae5bfa64cee9c5fb3a4168e87a01" \ + /tmp/autoe2e-checkpoint.pt + verify_s3_sha256 \ + "${BEVFORMER_CHECKPOINT_URI}" \ + "5585bc4d3ff8b396928cb92d91f773a2c57a81258f83cab0c668ebb2eb9d3307" \ + /tmp/bevformer-v2-epoch-24.pth + verify_s3_sha256 \ + "${PUBLISHED_SHARDS_URI%/}/manifest.json" \ + "${DATASET_MANIFEST_SHA256}" \ + /tmp/kitscenes-v3.3-manifest.json + - | + TRAINING_IMAGE_TAG="${TRAINING_IMAGE_TAG:-${IMAGE_TAG}}" + DATA_PREP_IMAGE_TAG="${DATA_PREP_IMAGE_TAG:-${IMAGE_TAG}}" + EVAL_IMAGE_TAG="${EVAL_IMAGE_TAG:-${IMAGE_TAG}}" + OFFLINE_RL_IMAGE_TAG="${OFFLINE_RL_IMAGE_TAG:-${IMAGE_TAG}}" + BEVFORMER_V2_IMAGE_TAG="${BEVFORMER_V2_IMAGE_TAG:-${IMAGE_TAG}}" + image_ref() { + REPOSITORY="$1" + TAG="$2" + DIGEST=$(aws ecr batch-get-image \ + --repository-name "auto-e2e/${REPOSITORY}" \ + --image-ids imageTag="${TAG}" \ + --query 'images[0].imageId.imageDigest' \ + --output text) + if [ -z "${DIGEST}" ] || [ "${DIGEST}" = "None" ]; then + echo "No digest found for auto-e2e/${REPOSITORY}:${TAG}" >&2 + return 1 + fi + printf '%s/auto-e2e/%s@%s' "${ECR_URL}" "${REPOSITORY}" "${DIGEST}" + } + AUTO_E2E_TRAINING_IMAGE="$( + image_ref training "${TRAINING_IMAGE_TAG}" + )" || exit 1 + AUTO_E2E_EVAL_IMAGE="$( + image_ref eval "${EVAL_IMAGE_TAG}" + )" || exit 1 + AUTO_E2E_OFFLINE_RL_IMAGE="$( + image_ref offline-rl "${OFFLINE_RL_IMAGE_TAG}" + )" || exit 1 + AUTO_E2E_DATA_PREP_IMAGE="$( + image_ref data-prep "${DATA_PREP_IMAGE_TAG}" + )" || exit 1 + AUTO_E2E_BEVFORMER_V2_IMAGE="$( + image_ref bevformer-v2 "${BEVFORMER_V2_IMAGE_TAG}" + )" || exit 1 + export AUTO_E2E_TRAINING_IMAGE AUTO_E2E_EVAL_IMAGE + export AUTO_E2E_OFFLINE_RL_IMAGE AUTO_E2E_DATA_PREP_IMAGE + export AUTO_E2E_BEVFORMER_V2_IMAGE + export ECR_PREFIX="${ECR_URL}" + export PYTHONPATH="${CODEBUILD_SRC_DIR}/Model:${CODEBUILD_SRC_DIR}:${PYTHONPATH:-}" + - | + cat > /tmp/flyte.yaml << EOF + admin: + endpoint: dns:///k8s-flyte-flyteadm-be92e7737c-cf0d1a0f3df10f77.elb.us-west-2.amazonaws.com:81 + insecure: true + EOF + - | + python3 - <<'PY' + import datetime + import json + import os + + timestamp = os.environ["PUBLICATION_TIMESTAMP"] + parsed = datetime.datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%SZ") + if parsed.strftime("%Y-%m-%dT%H:%M:%SZ") != timestamp: + raise ValueError("PUBLICATION_TIMESTAMP is not canonical UTC") + + common = { + "shard_dirs": [os.environ["PUBLISHED_SHARDS_URI"]], + "dataset": os.environ["DATASET"], + "dataset_version": os.environ["DATASET_VERSION"], + "dataset_manifest_sha256": os.environ[ + "DATASET_MANIFEST_SHA256" + ], + "artifacts_bucket": os.environ["ARTIFACTS_BUCKET"], + "publication_timestamp": timestamp, + "aws_region": os.environ["AWS_DEFAULT_REGION"], + } + native = { + **common, + "checkpoint": os.environ["AUTOE2E_CHECKPOINT_URI"], + "repository_revision": os.environ["REPOSITORY_REVISION"], + "batch_size": 2, + "num_workers": 0, + } + bevformer = { + **common, + "checkpoint": os.environ["BEVFORMER_CHECKPOINT_URI"], + "score_threshold": float(os.environ["SCORE_THRESHOLD"]), + } + for path, inputs in ( + ("/tmp/native-occupancy-inputs.json", native), + ("/tmp/bevformer-v2-occupancy-inputs.json", bevformer), + ): + with open(path, "w", encoding="utf-8") as stream: + json.dump(inputs, stream, sort_keys=True) + PY + - | + pyflyte --config /tmp/flyte.yaml run --remote \ + --project "${FLYTE_PROJECT}" \ + --domain "${FLYTE_DOMAIN}" \ + --image "${AUTO_E2E_TRAINING_IMAGE}" \ + Platform/pipelines/workflows.py \ + wf_precompute_semantic_occupancy \ + --inputs-file /tmp/native-occupancy-inputs.json + - | + pyflyte --config /tmp/flyte.yaml run --remote \ + --project "${FLYTE_PROJECT}" \ + --domain "${FLYTE_DOMAIN}" \ + --image "${AUTO_E2E_TRAINING_IMAGE}" \ + Platform/pipelines/workflows.py \ + wf_precompute_bevformer_v2_occupancy \ + --inputs-file /tmp/bevformer-v2-occupancy-inputs.json diff --git a/Platform/buildspec-register-distributed.yml b/Platform/buildspec-register-distributed.yml new file mode 100644 index 000000000..840cc3636 --- /dev/null +++ b/Platform/buildspec-register-distributed.yml @@ -0,0 +1,49 @@ +version: 0.2 + +env: + variables: + AWS_DEFAULT_REGION: us-west-2 + FLYTE_PROJECT: auto-e2e + FLYTE_DOMAIN: development + +phases: + install: + commands: + - > + pip install -q + flytekit==1.16.24 + flytekitplugins-ray==1.16.24 + ray==2.46.0 + numpy + torch + kubernetes + --extra-index-url https://download.pytorch.org/whl/cpu + build: + commands: + - test -n "${IMAGE_TAG}" + - ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) + - ECR_URL="${ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com" + - AUTO_E2E_RAY_STORAGE_PATH="s3://auto-e2e-platform-checkpoints-${ACCOUNT_ID}/ray-train" + - export AUTO_E2E_RAY_STORAGE_PATH + - > + IMAGE_DIGEST=$(aws ecr batch-get-image + --repository-name auto-e2e/training + --image-ids imageTag="${IMAGE_TAG}" + --query 'images[0].imageId.imageDigest' + --output text) + - test -n "${IMAGE_DIGEST}" && test "${IMAGE_DIGEST}" != "None" + - AUTO_E2E_TRAINING_IMAGE="${ECR_URL}/auto-e2e/training@${IMAGE_DIGEST}" + - export AUTO_E2E_TRAINING_IMAGE + - export PYTHONPATH="${CODEBUILD_SRC_DIR}/Model:${CODEBUILD_SRC_DIR}:${PYTHONPATH:-}" + - | + cat > /tmp/flyte.yaml << EOF + admin: + endpoint: dns:///k8s-flyte-flyteadm-be92e7737c-cf0d1a0f3df10f77.elb.us-west-2.amazonaws.com:81 + insecure: true + EOF + - > + pyflyte --config /tmp/flyte.yaml register + --project "${FLYTE_PROJECT}" + --domain "${FLYTE_DOMAIN}" + --image "${AUTO_E2E_TRAINING_IMAGE}" + Platform/pipelines/distributed_training.py diff --git a/Platform/buildspec-register.yml b/Platform/buildspec-register.yml index cdc1ae113..a9cf5c395 100644 --- a/Platform/buildspec-register.yml +++ b/Platform/buildspec-register.yml @@ -6,40 +6,65 @@ env: FLYTE_PROJECT: auto-e2e FLYTE_DOMAIN: development IMAGE_TAG: latest + TRAINING_IMAGE_TAG: "" + DATA_PREP_IMAGE_TAG: "" + EVAL_IMAGE_TAG: "" + OFFLINE_RL_IMAGE_TAG: "" + BEVFORMER_V2_IMAGE_TAG: "" phases: install: commands: # kubernetes: required to import the V1PodSpec/V1Volume models used by the # train/eval /dev/shm PodTemplate (#121 P0) at registration time. - - pip install -q flytekit==1.14.9 numpy torch kubernetes --extra-index-url https://download.pytorch.org/whl/cpu + - pip install -q flytekit==1.16.24 flytekitplugins-ray==1.16.24 ray==2.46.0 numpy torch kubernetes --extra-index-url https://download.pytorch.org/whl/cpu build: commands: - ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) - ECR_URL="${ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com" + - AUTO_E2E_RAY_STORAGE_PATH="s3://auto-e2e-platform-checkpoints-${ACCOUNT_ID}/ray-train" + - export AUTO_E2E_RAY_STORAGE_PATH - test -n "${IMAGE_TAG}" - export ECR_PREFIX="${ECR_URL}" - export PYTHONPATH="${CODEBUILD_SRC_DIR}/Model:${CODEBUILD_SRC_DIR}:${PYTHONPATH:-}" - | + TRAINING_IMAGE_TAG="${TRAINING_IMAGE_TAG:-${IMAGE_TAG}}" + DATA_PREP_IMAGE_TAG="${DATA_PREP_IMAGE_TAG:-${IMAGE_TAG}}" + EVAL_IMAGE_TAG="${EVAL_IMAGE_TAG:-${IMAGE_TAG}}" + OFFLINE_RL_IMAGE_TAG="${OFFLINE_RL_IMAGE_TAG:-${IMAGE_TAG}}" + BEVFORMER_V2_IMAGE_TAG="${BEVFORMER_V2_IMAGE_TAG:-${IMAGE_TAG}}" image_ref() { REPOSITORY="$1" + TAG="$2" DIGEST=$(aws ecr batch-get-image \ --repository-name "auto-e2e/${REPOSITORY}" \ - --image-ids imageTag="${IMAGE_TAG}" \ + --image-ids imageTag="${TAG}" \ --query 'images[0].imageId.imageDigest' \ --output text) if [ -z "${DIGEST}" ] || [ "${DIGEST}" = "None" ]; then - echo "No digest found for auto-e2e/${REPOSITORY}:${IMAGE_TAG}" >&2 + echo "No digest found for auto-e2e/${REPOSITORY}:${TAG}" >&2 return 1 fi printf '%s/auto-e2e/%s@%s' "${ECR_URL}" "${REPOSITORY}" "${DIGEST}" } - AUTO_E2E_TRAINING_IMAGE="$(image_ref training)" || exit 1 - AUTO_E2E_EVAL_IMAGE="$(image_ref eval)" || exit 1 - AUTO_E2E_OFFLINE_RL_IMAGE="$(image_ref offline-rl)" || exit 1 - AUTO_E2E_DATA_PREP_IMAGE="$(image_ref data-prep)" || exit 1 + AUTO_E2E_TRAINING_IMAGE="$( + image_ref training "${TRAINING_IMAGE_TAG}" + )" || exit 1 + AUTO_E2E_EVAL_IMAGE="$( + image_ref eval "${EVAL_IMAGE_TAG}" + )" || exit 1 + AUTO_E2E_OFFLINE_RL_IMAGE="$( + image_ref offline-rl "${OFFLINE_RL_IMAGE_TAG}" + )" || exit 1 + AUTO_E2E_DATA_PREP_IMAGE="$( + image_ref data-prep "${DATA_PREP_IMAGE_TAG}" + )" || exit 1 + AUTO_E2E_BEVFORMER_V2_IMAGE="$( + image_ref bevformer-v2 "${BEVFORMER_V2_IMAGE_TAG}" + )" || exit 1 export AUTO_E2E_TRAINING_IMAGE AUTO_E2E_EVAL_IMAGE export AUTO_E2E_OFFLINE_RL_IMAGE AUTO_E2E_DATA_PREP_IMAGE + export AUTO_E2E_BEVFORMER_V2_IMAGE - | cat > /tmp/flyte.yaml << EOF admin: @@ -52,3 +77,4 @@ phases: --domain $FLYTE_DOMAIN --image "${AUTO_E2E_TRAINING_IMAGE}" Platform/pipelines/workflows.py + Platform/pipelines/distributed_training.py diff --git a/Platform/buildspec-training.yml b/Platform/buildspec-training.yml new file mode 100644 index 000000000..3c1fda655 --- /dev/null +++ b/Platform/buildspec-training.yml @@ -0,0 +1,32 @@ +version: 0.2 + +env: + variables: + AWS_DEFAULT_REGION: us-west-2 + DOCKER_BUILDKIT: "1" + +phases: + pre_build: + commands: + - test -n "${IMAGE_TAG}" + - ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) + - ECR_URL="${ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com" + - aws ecr get-login-password --region "${AWS_DEFAULT_REGION}" | docker login --username AWS --password-stdin "${ECR_URL}" + - docker pull "${ECR_URL}/auto-e2e/training:latest" || true + build: + commands: + - > + DOCKER_BUILDKIT=1 docker build + --cache-from "${ECR_URL}/auto-e2e/training:latest" + --build-arg BUILDKIT_INLINE_CACHE=1 + -t "${ECR_URL}/auto-e2e/training:${IMAGE_TAG}" + -f Platform/docker/training/Dockerfile . + - docker push "${ECR_URL}/auto-e2e/training:${IMAGE_TAG}" + post_build: + commands: + - > + aws ecr describe-images + --repository-name auto-e2e/training + --image-ids imageTag="${IMAGE_TAG}" + --query 'imageDetails[0].{digest:imageDigest,pushed:imagePushedAt,size:imageSizeInBytes}' + --output json diff --git a/Platform/buildspec.yml b/Platform/buildspec.yml index f955fa7df..e9d584fd6 100644 --- a/Platform/buildspec.yml +++ b/Platform/buildspec.yml @@ -19,6 +19,7 @@ phases: - echo "=== Pre-pulling previous latests for layer cache ===" - (docker pull ${ECR_URL}/auto-e2e/training:latest || true) & - (docker pull ${ECR_URL}/auto-e2e/data-prep:latest || true) & + - (docker pull ${ECR_URL}/auto-e2e/bevformer-v2:latest || true) & - wait build: commands: @@ -47,6 +48,15 @@ phases: -t ${ECR_URL}/auto-e2e/data-prep:${IMAGE_TAG} -f Platform/docker/data-prep/Dockerfile . + # ── BEVFormer V2 ── pinned legacy CUDA/OpenMMLab runtime used only by + # occupancy publication. Keep it separate from the shared training image. + - echo "=== Building BEVFormer V2 occupancy image ===" + - DOCKER_BUILDKIT=1 docker build + --cache-from ${ECR_URL}/auto-e2e/bevformer-v2:latest + --build-arg BUILDKIT_INLINE_CACHE=1 + -t ${ECR_URL}/auto-e2e/bevformer-v2:${IMAGE_TAG} + -f Platform/docker/bevformer-v2/Dockerfile . + # ── Push ALL images in parallel now that every image is built. ECR dedups # blobs so the training/eval/offline-rl push is paid once, and data-prep # ships in parallel — much faster than the old sequential push. @@ -60,12 +70,15 @@ phases: PID_O=$! docker push ${ECR_URL}/auto-e2e/data-prep:${IMAGE_TAG} & PID_D=$! + docker push ${ECR_URL}/auto-e2e/bevformer-v2:${IMAGE_TAG} & + PID_B=$! # wait -n was returning early on some CodeBuild agents; wait on each PID # explicitly so a push failure is surfaced. wait $PID_T && echo "training pushed" || exit 1 wait $PID_E && echo "eval pushed" || exit 1 wait $PID_O && echo "offline-rl pushed" || exit 1 wait $PID_D && echo "data-prep pushed" || exit 1 + wait $PID_B && echo "bevformer-v2 pushed" || exit 1 cache: paths: diff --git a/Platform/docker/bevformer-v2/Dockerfile b/Platform/docker/bevformer-v2/Dockerfile new file mode 100644 index 000000000..d316b441d --- /dev/null +++ b/Platform/docker/bevformer-v2/Dockerfile @@ -0,0 +1,133 @@ +FROM nvidia/cuda:11.1.1-cudnn8-devel-ubuntu20.04 + +ARG DEBIAN_FRONTEND=noninteractive +ARG BEVFORMER_REVISION=66b65f3a1f58caf0507cb2a971b9c0e7f842376c +ARG DETECTRON2_REVISION=d1e04565d3bec8719335b88be9e9b961bf3ec464 +ARG MMDET3D_REVISION=f1107977dfd26155fc1f83779ee6535d2468f449 + +ENV CUDA_HOME=/usr/local/cuda +ENV FORCE_CUDA=1 +ENV PYTHONUNBUFFERED=1 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + git \ + libgl1 \ + libglib2.0-0 \ + ninja-build \ + python3.8 \ + python3.8-dev \ + python3-pip \ + && rm -rf /var/lib/apt/lists/* \ + && python3.8 -m pip install --no-cache-dir --upgrade \ + pip==23.3.2 \ + setuptools==59.5.0 \ + wheel==0.42.0 + +RUN python3.8 -m pip install --no-cache-dir \ + torch==1.9.1+cu111 \ + torchvision==0.10.1+cu111 \ + torchaudio==0.9.1 \ + -f https://download.pytorch.org/whl/torch_stable.html \ + && python3.8 -m pip install --no-cache-dir \ + opencv-python==4.5.5.64 \ + && python3.8 -m pip install --no-cache-dir \ + mmcv-full==1.4.0 \ + -f https://download.openmmlab.com/mmcv/dist/cu111/torch1.9.0/index.html \ + && python3.8 -m pip install --no-cache-dir \ + mmdet==2.14.0 \ + mmsegmentation==0.14.1 + +# CUDA 11.1 cannot emit sm_89 directly; PTX compiled for 8.6 forward-JITs on +# the L40S GPUs used by the g6e node pools. +ARG TORCH_CUDA_ARCH_LIST="8.6+PTX" +ENV TORCH_CUDA_ARCH_LIST=${TORCH_CUDA_ARCH_LIST} + +RUN git clone https://github.com/open-mmlab/mmdetection3d.git /opt/mmdetection3d \ + && git -C /opt/mmdetection3d checkout "${MMDET3D_REVISION}" \ + && python3.8 -m pip install --no-cache-dir --no-deps /opt/mmdetection3d + +RUN git clone https://github.com/facebookresearch/detectron2.git /opt/detectron2 \ + && git -C /opt/detectron2 checkout "${DETECTRON2_REVISION}" \ + && python3.8 -m pip install --no-cache-dir --no-deps /opt/detectron2 + +# mmdetection3d and Detectron2 are source-installed with --no-deps above. +# Keep their runtime requirements explicit alongside Flyte's S3 dependency set. +RUN python3.8 -m pip install --no-cache-dir \ + aiobotocore==2.17.0 \ + black==21.4b2 \ + boto3==1.35.93 \ + cloudpickle==2.2.1 \ + einops==0.6.1 \ + flytekit==1.13.15 \ + fsspec==2024.6.1 \ + future==1.0.0 \ + fvcore==0.1.5.post20221221 \ + hydra-core==1.1.2 \ + iopath==0.1.9 \ + kubernetes==28.1.0 \ + lyft-dataset-sdk==0.0.8 \ + matplotlib==3.5.2 \ + networkx==2.2 \ + numba==0.48.0 \ + numpy==1.19.5 \ + nuscenes-devkit==1.1.9 \ + omegaconf==2.1.2 \ + pandas==1.4.4 \ + Pillow==9.5.0 \ + plyfile==0.7.4 \ + pydot==1.4.2 \ + pycocotools==2.0.7 \ + pyquaternion==0.9.9 \ + s3fs==2024.6.1 \ + scikit-image==0.19.3 \ + scipy==1.9.3 \ + seaborn==0.12.2 \ + tabulate==0.9.0 \ + terminaltables==3.1.10 \ + tensorboard==2.13.0 \ + termcolor==2.4.0 \ + timm==0.6.13 \ + tqdm==4.66.5 \ + trimesh==2.35.39 \ + typing-extensions==4.5.0 \ + yacs==0.1.8 + +# Remove an unused Tk constant import that breaks the headless runtime. +RUN git clone https://github.com/fundamentalvision/BEVFormer.git /opt/BEVFormer \ + && git -C /opt/BEVFormer checkout "${BEVFORMER_REVISION}" \ + && test "$(git -C /opt/BEVFormer rev-parse HEAD)" = "${BEVFORMER_REVISION}" \ + && sed -i '/from tkinter\.messagebox import NO/d' \ + /opt/BEVFormer/projects/mmdet3d_plugin/bevformer/detectors/bevformer_fp16.py \ + && ! grep -Fq "from tkinter.messagebox import NO" \ + /opt/BEVFormer/projects/mmdet3d_plugin/bevformer/detectors/bevformer_fp16.py + +WORKDIR /workspace + +COPY Model/ /workspace/Model/ +COPY Platform/pipelines/ /workspace/Platform/pipelines/ + +ENV PYTHONPATH=/opt/BEVFormer:/workspace/Model:/workspace + +RUN python3.8 -m pip check \ + && python3.8 -c \ + "import cv2, importlib, mmcv, mmdet, mmseg, mmdet3d, torch; \ +from botocore.session import Session; \ +from Platform.pipelines import bevformer_v2_occupancy, bevformer_v2_runtime, workflows; \ +from mmcv import Config; \ +from mmdet3d.models import build_model; \ +assert bevformer_v2_occupancy.BEVFORMER_V2_WEIGHT_SHA256; \ +assert bevformer_v2_runtime.BEVFORMER_V2_IMAGE_WIDTH == 640; \ +assert workflows.precompute_bevformer_v2_occupancy_artifacts.name; \ +assert torch.__version__.startswith('1.9.1'); \ +assert cv2.__version__ == '4.5.5'; \ +assert mmcv.__version__ == '1.4.0'; \ +assert mmdet.__version__ == '2.14.0'; \ +assert mmseg.__version__ == '0.14.1'; \ +assert 'IfNoneMatch' in Session().get_service_model('s3').operation_model('PutObject').input_shape.members; \ +importlib.import_module('projects.mmdet3d_plugin'); \ +config=Config.fromfile('/opt/BEVFormer/projects/configs/bevformerv2/bevformerv2-r50-t8-24ep.py'); \ +config.model.pretrained=None; \ +config.model.train_cfg=None; \ +build_model(config.model, test_cfg=config.get('test_cfg'))" diff --git a/Platform/docker/data-prep/Dockerfile b/Platform/docker/data-prep/Dockerfile index 1c192c885..97a3783dc 100644 --- a/Platform/docker/data-prep/Dockerfile +++ b/Platform/docker/data-prep/Dockerfile @@ -28,6 +28,16 @@ RUN git lfs install --system \ && pip install --no-cache-dir ".[map]" \ && rm -rf /tmp/kitscenes +# nuPlan is installed from one reviewed revision without its obsolete global +# requirements lock. The minimal Python 3.12-compatible runtime dependencies +# are pinned below together with the rest of the data-prep environment. +ARG NUPLAN_DEVKIT_REVISION=e9241677997dd86bfc0bcd44817ab04fe631405b +RUN GIT_LFS_SKIP_SMUDGE=1 git clone \ + https://github.com/motional/nuplan-devkit.git /tmp/nuplan-devkit \ + && git -C /tmp/nuplan-devkit checkout "${NUPLAN_DEVKIT_REVISION}" \ + && pip install --no-cache-dir --no-deps /tmp/nuplan-devkit \ + && rm -rf /tmp/nuplan-devkit + # KITScenes requires NumPy 1.x, while lerobot-dataset 0.5.0's package metadata # requires NumPy 2.x even though its dataset runtime remains NumPy 1-compatible. # Install its non-conflicting runtime dependencies explicitly, then install the @@ -49,6 +59,14 @@ RUN pip install --no-cache-dir \ "deepdiff>=7.0,<9.0" \ "imageio[ffmpeg]>=2.34,<3.0" \ "jsonlines>=4.0,<5.0" \ + networkx \ + "osmium==4.3.1" \ + "geopandas>=0.14,<1.0" \ + "hydra-core>=1.3,<1.4" \ + pyquaternion \ + "SQLAlchemy>=1.4.54,<2.0" \ + retry \ + ujson \ "packaging>=24.2,<26.0" \ "termcolor>=2.4,<4.0" \ timm \ diff --git a/Platform/docker/training/Dockerfile b/Platform/docker/training/Dockerfile index f6c912866..ba9e1430e 100644 --- a/Platform/docker/training/Dockerfile +++ b/Platform/docker/training/Dockerfile @@ -2,11 +2,20 @@ FROM pytorch/pytorch:2.4.1-cuda12.1-cudnn9-runtime WORKDIR /workspace +ENV HF_HOME=/opt/model-cache/huggingface +ENV TORCH_HOME=/opt/model-cache/torch + +RUN apt-get update \ + && apt-get install -y --no-install-recommends wget \ + && rm -rf /var/lib/apt/lists/* + RUN pip install --no-cache-dir \ timm==1.0.27 \ mlflow-skinny==2.22.0 \ webdataset \ flytekit==1.16.24 \ + flytekitplugins-ray==1.16.24 \ + "ray[train]==2.46.0" \ fsspec==2026.6.0 \ s3fs==2026.6.0 \ gcsfs==2026.7.0 \ @@ -17,8 +26,14 @@ RUN pip install --no-cache-dir \ pyproj==3.7.2 \ boto3==1.43.0 +# Cache the exact Stage A backbone in the immutable training image so all +# workers avoid identical runtime downloads. +RUN python -c \ + "import timm; timm.create_model('swinv2_tiny_window8_256', pretrained=True, features_only=True)" + COPY Model/ /workspace/Model/ COPY Platform/pipelines/ /workspace/Platform/pipelines/ ENV PYTHONPATH=/workspace/Model:/workspace ENV PYTHONUNBUFFERED=1 +ENV RAY_TRAIN_V2_ENABLED=1 diff --git a/Platform/helm-values/flyte-core-eks.yaml b/Platform/helm-values/flyte-core-eks.yaml index 2532e2efc..f2ec54ec6 100644 --- a/Platform/helm-values/flyte-core-eks.yaml +++ b/Platform/helm-values/flyte-core-eks.yaml @@ -11,6 +11,7 @@ userSettings: logGroup: redisHostUrl: redisHostKey: + controlPlaneRoleName: # # FLYTEADMIN @@ -20,11 +21,10 @@ flyteadmin: replicaCount: 2 # -- IAM role for SA: https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html serviceAccount: - # -- If the service account is created by you, make this false, else a new service account will be created and the iam-role-flyte will be added - # you can change the name of this role + # -- If the service account is created by you, make this false. create: true annotations: - eks.amazonaws.com/role-arn: arn:aws:iam::{{ .Values.userSettings.accountNumber }}:role/iam-role-flyte + eks.amazonaws.com/role-arn: arn:aws:iam::{{ .Values.userSettings.accountNumber }}:role/{{ .Values.userSettings.controlPlaneRoleName }} serviceMonitor: enabled: false @@ -61,7 +61,7 @@ datacatalog: # -- If the service account is created by you, make this false create: true annotations: - eks.amazonaws.com/role-arn: arn:aws:iam::{{ .Values.userSettings.accountNumber }}:role/iam-role-flyte + eks.amazonaws.com/role-arn: arn:aws:iam::{{ .Values.userSettings.accountNumber }}:role/{{ .Values.userSettings.controlPlaneRoleName }} resources: limits: cpu: 1 @@ -93,7 +93,7 @@ flytepropeller: # -- If the service account is created by you, make this false create: true annotations: - eks.amazonaws.com/role-arn: arn:aws:iam::{{ .Values.userSettings.accountNumber }}:role/iam-role-flyte + eks.amazonaws.com/role-arn: arn:aws:iam::{{ .Values.userSettings.accountNumber }}:role/{{ .Values.userSettings.controlPlaneRoleName }} resources: limits: cpu: 1 @@ -298,6 +298,7 @@ configmap: - container - sidecar - k8s-array + - ray - connector-service - echo # - sagemaker_hyperparameter_tuning @@ -307,6 +308,7 @@ configmap: container: container sidecar: sidecar container_array: k8s-array + ray: ray # sagemaker_custom_training_task: sagemaker_custom_training # sagemaker_custom_training_job_task: sagemaker_custom_training @@ -377,6 +379,8 @@ cluster_resource_manager: value: "5" - projectQuotaMemory: value: "4000Mi" + - projectQuotaGpu: + value: "1" - defaultIamRole: value: "arn:aws:iam::{{ .Values.userSettings.accountNumber }}:role/flyte-user-role" - staging: @@ -384,6 +388,8 @@ cluster_resource_manager: value: "2" - projectQuotaMemory: value: "3000Mi" + - projectQuotaGpu: + value: "1" - defaultIamRole: value: "arn:aws:iam::{{ .Values.userSettings.accountNumber }}:role/flyte-user-role" - development: @@ -394,6 +400,8 @@ cluster_resource_manager: value: "1000" - projectQuotaMemory: value: "8Ti" + - projectQuotaGpu: + value: "8" - defaultIamRole: value: "arn:aws:iam::{{ .Values.userSettings.accountNumber }}:role/flyte-user-role" @@ -431,8 +439,10 @@ cluster_resource_manager: hard: limits.cpu: {{ projectQuotaCpu }} limits.memory: {{ projectQuotaMemory }} + limits.nvidia.com/gpu: {{ projectQuotaGpu }} requests.cpu: {{ projectQuotaCpu }} requests.memory: {{ projectQuotaMemory }} + requests.nvidia.com/gpu: {{ projectQuotaGpu }} diff --git a/Platform/infra/main.tf b/Platform/infra/main.tf index e8af43fc7..3f5277c3e 100644 --- a/Platform/infra/main.tf +++ b/Platform/infra/main.tf @@ -60,12 +60,20 @@ module "training_operator" { depends_on = [module.eks] } +module "kuberay" { + source = "./modules/kuberay" + + cluster_name = var.cluster_name + + depends_on = [module.eks] +} + module "kueue" { source = "./modules/kueue" cluster_name = var.cluster_name - depends_on = [module.training_operator] + depends_on = [module.training_operator, module.kuberay] } module "mlflow" { @@ -160,6 +168,10 @@ output "overlay_launch_project" { value = module.codebuild.overlay_launch_project } +output "occupancy_publish_project" { + value = module.codebuild.occupancy_publish_project +} + # --- UI Exposure: CloudFront + VPC Origin → Internal NLB (K8s managed) --- # NLB ARNs/DNS are passed as variables since K8s Service creates them. # After first deploy, run post-apply to create NLB Services, then set these vars. diff --git a/Platform/infra/modules/codebuild/main.tf b/Platform/infra/modules/codebuild/main.tf index e5154e91f..2baded341 100644 --- a/Platform/infra/modules/codebuild/main.tf +++ b/Platform/infra/modules/codebuild/main.tf @@ -61,6 +61,21 @@ resource "aws_iam_role_policy" "codebuild" { Action = ["s3:GetObject", "s3:PutObject", "s3:GetBucketLocation"] Resource = ["${aws_s3_bucket.cache.arn}", "${aws_s3_bucket.cache.arn}/*"] }, + { + Effect = "Allow" + Action = ["s3:GetObject", "s3:GetBucketLocation"] + Resource = [ + "arn:aws:s3:::${var.cluster_name}-checkpoints-${local.account_id}", + "arn:aws:s3:::${var.cluster_name}-checkpoints-${local.account_id}/*", + "arn:aws:s3:::${var.cluster_name}-datasets-${local.account_id}", + "arn:aws:s3:::${var.cluster_name}-datasets-${local.account_id}/*", + ] + }, + { + Effect = "Allow" + Action = ["s3:PutObject"] + Resource = ["arn:aws:s3:::${var.cluster_name}-datasets-${local.account_id}/*/odd/configs/*"] + }, ] }) } @@ -227,6 +242,46 @@ output "overlay_launch_project" { value = aws_codebuild_project.overlay_launch.name } +# --- Semantic occupancy publication (VPC-local Flyte client) --- + +resource "aws_codebuild_project" "occupancy_publish" { + name = "${var.cluster_name}-occupancy-publish" + service_role = aws_iam_role.codebuild.arn + + artifacts { type = "NO_ARTIFACTS" } + + environment { + compute_type = "BUILD_GENERAL1_SMALL" + image = "aws/codebuild/amazonlinux-x86_64-standard:5.0" + type = "LINUX_CONTAINER" + image_pull_credentials_type = "CODEBUILD" + } + + source { + type = "S3" + location = "${aws_s3_bucket.cache.bucket}/source.zip" + buildspec = "Platform/buildspec-publish-occupancy.yml" + } + + vpc_config { + vpc_id = var.vpc_id + subnets = var.private_subnet_ids + security_group_ids = [aws_security_group.flyte_register.id] + } + + logs_config { + cloudwatch_logs { + group_name = "/codebuild/${var.cluster_name}-occupancy-publish" + } + } + + tags = { Purpose = "semantic-occupancy-publication" } +} + +output "occupancy_publish_project" { + value = aws_codebuild_project.occupancy_publish.name +} + # Allow CodeBuild flyte-register to reach flyteadmin (gRPC port 81) resource "aws_security_group_rule" "codebuild_to_flyteadmin" { type = "ingress" diff --git a/Platform/infra/modules/ecr/main.tf b/Platform/infra/modules/ecr/main.tf index a0b40deff..5a45566ce 100644 --- a/Platform/infra/modules/ecr/main.tf +++ b/Platform/infra/modules/ecr/main.tf @@ -1,7 +1,7 @@ variable "environment" { type = string } locals { - repositories = ["auto-e2e/training", "auto-e2e/data-prep", "auto-e2e/eval", "auto-e2e/offline-rl"] + repositories = ["auto-e2e/training", "auto-e2e/data-prep", "auto-e2e/eval", "auto-e2e/offline-rl", "auto-e2e/bevformer-v2"] } resource "aws_ecr_repository" "this" { diff --git a/Platform/infra/modules/flyte/main.tf b/Platform/infra/modules/flyte/main.tf index 4226225e7..78f0a1436 100644 --- a/Platform/infra/modules/flyte/main.tf +++ b/Platform/infra/modules/flyte/main.tf @@ -101,6 +101,15 @@ resource "aws_iam_role_policy" "flyte_user_s3" { "arn:aws:s3:::${var.datasets_bucket}", ] }, + # S3 multipart copy authorizes the source through the Flyte task role. + # Keep this read-only grant scoped to the official nuPlan v1.1 prefix. + { + Effect = "Allow" + Action = ["s3:GetObject"] + Resource = [ + "arn:aws:s3:::motional-nuplan/public/nuplan-v1.1/*", + ] + }, { Effect = "Allow" Action = ["dynamodb:GetItem", "dynamodb:PutItem"] @@ -163,6 +172,10 @@ resource "helm_release" "flyte" { name = "userSettings.accountRegion" value = var.region } + set { + name = "userSettings.controlPlaneRoleName" + value = "${var.cluster_name}-s3-access" + } set { name = "userSettings.certificateArn" value = "" @@ -191,6 +204,22 @@ resource "helm_release" "flyte" { name = "flyteadmin.serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" value = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${var.cluster_name}-s3-access" } + set { + name = "flytepropeller.serviceAccount.name" + value = "flytepropeller" + } + set { + name = "flytepropeller.serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" + value = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${var.cluster_name}-s3-access" + } + set { + name = "datacatalog.serviceAccount.name" + value = "datacatalog" + } + set { + name = "datacatalog.serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" + value = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${var.cluster_name}-s3-access" + } set { name = "db.admin.database.username" value = "pgadmin" diff --git a/Platform/infra/modules/kuberay/main.tf b/Platform/infra/modules/kuberay/main.tf new file mode 100644 index 000000000..6b3fb9833 --- /dev/null +++ b/Platform/infra/modules/kuberay/main.tf @@ -0,0 +1,22 @@ +variable "cluster_name" { type = string } + +resource "helm_release" "kuberay_operator" { + name = "kuberay-operator" + repository = "https://ray-project.github.io/kuberay-helm/" + chart = "kuberay-operator" + version = "1.4.2" + namespace = "kuberay-system" + create_namespace = true + timeout = 600 + wait = true + + set { + name = "leaderElectionEnabled" + value = "true" + } + + set { + name = "metrics.enabled" + value = "true" + } +} diff --git a/Platform/infra/modules/kueue/main.tf b/Platform/infra/modules/kueue/main.tf index c2260ea83..350cf84ff 100644 --- a/Platform/infra/modules/kueue/main.tf +++ b/Platform/infra/modules/kueue/main.tf @@ -16,6 +16,10 @@ resource "helm_release" "kueue" { name = "controller.manager.configuration.integrations.frameworks[1]" value = "kubeflow.org/pytorchjob" } + set { + name = "controller.manager.configuration.integrations.frameworks[2]" + value = "ray.io/rayjob" + } values = [file("${path.module}/../../../helm-values/kueue.yaml")] } diff --git a/Platform/infra/modules/storage/main.tf b/Platform/infra/modules/storage/main.tf index 88a6d7982..3b101e922 100644 --- a/Platform/infra/modules/storage/main.tf +++ b/Platform/infra/modules/storage/main.tf @@ -13,6 +13,8 @@ variable "pod_identity_associations" { default = [ { namespace = "auto-e2e-training", service_account = "training-sa" }, { namespace = "flyte", service_account = "flyteadmin" }, + { namespace = "flyte", service_account = "flytepropeller" }, + { namespace = "flyte", service_account = "datacatalog" }, { namespace = "mlflow", service_account = "mlflow" }, ] } @@ -72,7 +74,11 @@ resource "aws_iam_role" "s3_access" { Action = "sts:AssumeRoleWithWebIdentity" Condition = { StringLike = { - "${var.oidc_provider_url}:sub" = "system:serviceaccount:flyte:flyteadmin" + "${var.oidc_provider_url}:sub" = [ + "system:serviceaccount:flyte:flyteadmin", + "system:serviceaccount:flyte:flytepropeller", + "system:serviceaccount:flyte:datacatalog", + ] } } }] : [] diff --git a/Platform/infra/post-apply.sh b/Platform/infra/post-apply.sh index 5fbdf7f06..f53f68e5c 100755 --- a/Platform/infra/post-apply.sh +++ b/Platform/infra/post-apply.sh @@ -15,6 +15,10 @@ PROFILE="${AWS_PROFILE:-autowarefoundation}" REGION="${AWS_REGION:-us-west-2}" CLUSTER="${EKS_CLUSTER:-auto-e2e-platform}" CONTAINER_CLI="${CONTAINER_CLI:-finch}" # finch or docker +ACCOUNT=$(aws sts get-caller-identity \ + --profile "$PROFILE" \ + --query Account \ + --output text) echo "=== 1. Update kubeconfig ===" aws eks update-kubeconfig --name "$CLUSTER" --region "$REGION" --profile "$PROFILE" @@ -22,14 +26,15 @@ aws eks update-kubeconfig --name "$CLUSTER" --region "$REGION" --profile "$PROFI echo "=== 2. Verify cluster access ===" kubectl get nodepools -echo "=== 3. Apply GPU NodePool + warm-node keeper ===" +echo "=== 3. Apply GPU NodeClass, NodePool, and warm-node keeper ===" +sed "s/REPLACE_WITH_AWS_ACCOUNT_ID/${ACCOUNT}/g" \ + ../k8s/karpenter-nodepools/gpu-nodeclass.yaml | kubectl apply -f - kubectl apply -f ../k8s/karpenter-nodepools/gpu-nodepool.yaml kubectl apply -f ../k8s/gpu-node-keeper.yaml echo "Waiting for a GPU node to register (Karpenter provisions g6e)..." kubectl wait --for=condition=Ready node -l workload-type=gpu-training --timeout=600s echo "=== 4. ECR login ===" -ACCOUNT=$(aws sts get-caller-identity --profile "$PROFILE" --query Account --output text) ECR_URL="${ACCOUNT}.dkr.ecr.${REGION}.amazonaws.com" aws ecr get-login-password --region "$REGION" --profile "$PROFILE" | \ "$CONTAINER_CLI" login --username AWS --password-stdin "$ECR_URL" diff --git a/Platform/k8s/karpenter-nodepools/gpu-nodeclass.yaml b/Platform/k8s/karpenter-nodepools/gpu-nodeclass.yaml new file mode 100644 index 000000000..45d617bf1 --- /dev/null +++ b/Platform/k8s/karpenter-nodepools/gpu-nodeclass.yaml @@ -0,0 +1,25 @@ +apiVersion: eks.amazonaws.com/v1 +kind: NodeClass +metadata: + name: auto-e2e-gpu-training +spec: + role: auto-e2e-platform-node-role + subnetSelectorTerms: + - tags: + Name: auto-e2e-platform-private-us-west-2b + securityGroupSelectorTerms: + - tags: + aws:eks:cluster-name: auto-e2e-platform + capacityReservationSelectorTerms: + - ownerID: "REPLACE_WITH_AWS_ACCOUNT_ID" + tags: + Name: auto-e2e-distributed-training + placementGroupSelector: + name: auto-e2e-distributed-training-pg + ephemeralStorage: + size: 200Gi + iops: 3000 + throughput: 125 + networkPolicy: DefaultAllow + networkPolicyEventLogs: Disabled + snatPolicy: Random diff --git a/Platform/k8s/karpenter-nodepools/gpu-nodepool.yaml b/Platform/k8s/karpenter-nodepools/gpu-nodepool.yaml index d08edfdb8..9c57009a0 100644 --- a/Platform/k8s/karpenter-nodepools/gpu-nodepool.yaml +++ b/Platform/k8s/karpenter-nodepools/gpu-nodepool.yaml @@ -11,7 +11,7 @@ spec: nodeClassRef: group: eks.amazonaws.com kind: NodeClass - name: default + name: auto-e2e-gpu-training requirements: - key: node.kubernetes.io/instance-type operator: In @@ -21,7 +21,7 @@ spec: values: ["amd64"] - key: karpenter.sh/capacity-type operator: In - values: ["on-demand"] + values: ["reserved"] # Pinned to the AZ where the GPU ODCR is held (capacity-constrained) - key: topology.kubernetes.io/zone operator: In @@ -30,10 +30,10 @@ spec: - key: nvidia.com/gpu effect: NoSchedule limits: - cpu: "64" - memory: 256Gi - nodes: "1" - nvidia.com/gpu: "4" + cpu: "128" + memory: 1Ti + nodes: "8" + nvidia.com/gpu: "8" --- apiVersion: karpenter.sh/v1 kind: NodePool diff --git a/Platform/k8s/kueue-config/kueue-objects.yaml b/Platform/k8s/kueue-config/kueue-objects.yaml index 336e3873b..9cdbcc565 100644 --- a/Platform/k8s/kueue-config/kueue-objects.yaml +++ b/Platform/k8s/kueue-config/kueue-objects.yaml @@ -1,61 +1,88 @@ # Kueue cluster-scoped objects for GPU queue management. -# Apply AFTER Kueue Helm install. +# Apply after the KubeRay and Kueue CRDs are installed. --- -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ResourceFlavor metadata: - name: g6e-l40s + name: default-flavor +spec: {} +--- +apiVersion: kueue.x-k8s.io/v1beta2 +kind: ResourceFlavor +metadata: + name: gpu-flavor spec: nodeLabels: - node.kubernetes.io/instance-type: g6e.4xlarge + workload-type: gpu-training tolerations: - key: nvidia.com/gpu operator: Exists effect: NoSchedule --- -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: WorkloadPriorityClass metadata: name: research-low value: 1000 -description: "Default priority for research/sweep jobs" +description: "Default priority for research and canary jobs" --- -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: WorkloadPriorityClass metadata: name: production-high value: 10000 -description: "High priority for production training (preempts research)" +description: "High priority for production training" --- -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: - name: gpu-cq + name: training-queue spec: namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: auto-e2e-training + matchExpressions: + - key: kubernetes.io/metadata.name + operator: In + values: + - auto-e2e-development + - auto-e2e-training resourceGroups: - - coveredResources: ["cpu", "memory", "nvidia.com/gpu"] + - coveredResources: + - cpu + - memory flavors: - - name: g6e-l40s + - name: default-flavor resources: - - name: "cpu" - nominalQuota: "14" - - name: "memory" - nominalQuota: "120Gi" - - name: "nvidia.com/gpu" - nominalQuota: "1" + - name: cpu + nominalQuota: "40" + - name: memory + nominalQuota: 160Gi + - coveredResources: + - nvidia.com/gpu + flavors: + - name: gpu-flavor + resources: + - name: nvidia.com/gpu + nominalQuota: "8" preemption: withinClusterQueue: LowerPriority reclaimWithinCohort: Never --- -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: LocalQueue metadata: - name: gpu-queue + name: training namespace: auto-e2e-training annotations: kueue.x-k8s.io/default-queue: "true" spec: - clusterQueue: gpu-cq + clusterQueue: training-queue +--- +apiVersion: kueue.x-k8s.io/v1beta2 +kind: LocalQueue +metadata: + name: training + namespace: auto-e2e-development + annotations: + kueue.x-k8s.io/default-queue: "true" +spec: + clusterQueue: training-queue diff --git a/Platform/k8s/rayjob-templates/ddp-smoke-4.yaml b/Platform/k8s/rayjob-templates/ddp-smoke-4.yaml new file mode 100644 index 000000000..4bcf26b7a --- /dev/null +++ b/Platform/k8s/rayjob-templates/ddp-smoke-4.yaml @@ -0,0 +1,126 @@ +apiVersion: ray.io/v1 +kind: RayJob +metadata: + name: auto-e2e-ddp-smoke-4 + namespace: auto-e2e-development + labels: + kueue.x-k8s.io/queue-name: training + kueue.x-k8s.io/priority-class: research-low +spec: + suspend: true + shutdownAfterJobFinishes: true + ttlSecondsAfterFinished: 300 + backoffLimit: 0 + entrypoint: >- + python -m distributed_training.ray_smoke + --num-workers 4 + --steps 4 + --storage-path REPLACE_WITH_RAY_STORAGE_PATH + --run-name auto-e2e-ddp-smoke-4 + rayClusterSpec: + rayVersion: "2.46.0" + enableInTreeAutoscaling: false + headGroupSpec: + rayStartParams: + dashboard-host: "0.0.0.0" + num-cpus: "0" + template: + metadata: + labels: + auto-e2e.training/role: ray-head + annotations: + karpenter.sh/do-not-disrupt: "true" + spec: + serviceAccountName: default + containers: + - name: ray-head + image: REPLACE_WITH_TRAINING_IMAGE + imagePullPolicy: Always + env: + - name: AWS_DEFAULT_REGION + value: us-west-2 + - name: RAY_TRAIN_V2_ENABLED + value: "1" + ports: + - name: gcs-server + containerPort: 6379 + - name: dashboard + containerPort: 8265 + - name: client + containerPort: 10001 + - name: metrics + containerPort: 8080 + resources: + requests: + cpu: "2" + memory: 8Gi + limits: + cpu: "2" + memory: 8Gi + volumeMounts: + - name: dshm + mountPath: /dev/shm + volumes: + - name: dshm + emptyDir: + medium: Memory + sizeLimit: 2Gi + workerGroupSpecs: + - groupName: gpu-workers + replicas: 4 + minReplicas: 4 + maxReplicas: 4 + rayStartParams: + num-cpus: "4" + num-gpus: "1" + template: + metadata: + labels: + auto-e2e.training/role: ray-gpu-worker + annotations: + karpenter.sh/do-not-disrupt: "true" + spec: + serviceAccountName: default + nodeSelector: + workload-type: gpu-training + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + auto-e2e.training/role: ray-gpu-worker + topologyKey: kubernetes.io/hostname + containers: + - name: ray-worker + image: REPLACE_WITH_TRAINING_IMAGE + imagePullPolicy: Always + env: + - name: AWS_DEFAULT_REGION + value: us-west-2 + - name: RAY_TRAIN_V2_ENABLED + value: "1" + - name: NCCL_DEBUG + value: INFO + - name: TORCH_DISTRIBUTED_DEBUG + value: DETAIL + resources: + requests: + cpu: "4" + memory: 16Gi + nvidia.com/gpu: "1" + limits: + cpu: "4" + memory: 16Gi + nvidia.com/gpu: "1" + volumeMounts: + - name: dshm + mountPath: /dev/shm + volumes: + - name: dshm + emptyDir: + medium: Memory + sizeLimit: 8Gi diff --git a/Platform/pipelines/bevformer_v2_occupancy.py b/Platform/pipelines/bevformer_v2_occupancy.py new file mode 100644 index 000000000..097180792 --- /dev/null +++ b/Platform/pipelines/bevformer_v2_occupancy.py @@ -0,0 +1,347 @@ +"""Pure KITScenes adaptation helpers for BEVFormer V2 detections. + +The published BEVFormer V2 checkpoint is a 3-D detector, not an occupancy +segmentation model. This module therefore rasterizes only the ground-plane +footprints of predicted boxes. It never invents road, map, or teacher labels. +""" + +from __future__ import annotations + +import dataclasses +import math +from collections.abc import Mapping, Sequence + +import numpy as np + +from navigation.geometry import ( + AUTOE2E_NAVIGATION_GEOMETRY, + NavigationRasterGeometry, +) + +BEVFORMER_V2_REPOSITORY = "https://github.com/fundamentalvision/BEVFormer" +BEVFORMER_V2_REVISION = "66b65f3a1f58caf0507cb2a971b9c0e7f842376c" +BEVFORMER_V2_CONFIG_NAME = "bevformerv2-r50-t8-24ep.py" +BEVFORMER_V2_WEIGHT_SHA256 = ( + "5585bc4d3ff8b396928cb92d91f773a2c57a81258f83cab0c668ebb2eb9d3307" +) +BEVFORMER_V2_WEIGHT_SOURCE_URL = ( + "https://drive.google.com/drive/folders/1Ml_usx5BNx43CFH1Di2OTazuzSyAlBto" +) +BEVFORMER_V2_CODE_LICENSE_SPDX = "Apache-2.0" +BEVFORMER_V2_WEIGHT_LICENSE_SPDX = "NOASSERTION" +BEVFORMER_V2_TRAINING_DATA_LICENSE_SPDX = "CC-BY-NC-SA-4.0" +BEVFORMER_V2_HEAD_VERSION = "bevformer-v2-r50-t8-box-raster-v1" +BEVFORMER_V2_ARTIFACT_KIND = "detection-derived-occupancy" +BEVFORMER_V2_FRAMES = (-7, -6, -5, -4, -3, -2, -1, 0) +BEVFORMER_V2_SOURCE_HZ = 2 +KITSCENES_SOURCE_HZ = 10 + +SEMANTIC_CLASS_NAMES = ( + "drivable_area", + "lane_area", + "intersection", + "crosswalk", + "stop_line", + "vehicle", + "vulnerable_road_user", + "other_obstacle", +) + +_BEVFORMER_CLASS_TO_SEMANTIC = { + "barrier": "other_obstacle", + "bicycle": "vulnerable_road_user", + "bus": "vehicle", + "car": "vehicle", + "construction_vehicle": "vehicle", + "motorcycle": "vulnerable_road_user", + "pedestrian": "vulnerable_road_user", + "traffic_cone": "other_obstacle", + "trailer": "vehicle", + "truck": "vehicle", +} + +BEVFORMER_V2_SUPPORTED_SEMANTIC_CLASSES = tuple( + sorted(set(_BEVFORMER_CLASS_TO_SEMANTIC.values())) +) + + +@dataclasses.dataclass(frozen=True) +class DetectionBox: + """One BEVFormer box in the current KITScenes top-lidar FLU frame.""" + + class_name: str + score: float + center_x_m: float + center_y_m: float + length_m: float + width_m: float + yaw_rad: float + + def __post_init__(self) -> None: + numeric = ( + self.score, + self.center_x_m, + self.center_y_m, + self.length_m, + self.width_m, + self.yaw_rad, + ) + if not all(math.isfinite(value) for value in numeric): + raise ValueError("detection box values must be finite") + if self.class_name not in _BEVFORMER_CLASS_TO_SEMANTIC: + raise ValueError(f"unsupported BEVFormer class {self.class_name!r}") + if not 0.0 <= self.score <= 1.0: + raise ValueError("detection score must be in [0,1]") + if self.length_m <= 0.0 or self.width_m <= 0.0: + raise ValueError("detection dimensions must be positive") + + +def semantic_class_for_detection(class_name: str) -> str: + """Return the disclosed semantic class for one official detector class.""" + try: + return _BEVFORMER_CLASS_TO_SEMANTIC[class_name] + except KeyError as error: + raise ValueError( + f"unsupported BEVFormer class {class_name!r}" + ) from error + + +def temporal_frame_indices( + current_frame_index: int, + *, + scene_start_index: int = 0, + source_hz: int = KITSCENES_SOURCE_HZ, + model_hz: int = BEVFORMER_V2_SOURCE_HZ, + model_frames: Sequence[int] = BEVFORMER_V2_FRAMES, +) -> dict[int, int]: + """Map BEVFormer temporal offsets to KITScenes 10 Hz frame indices. + + Missing early history is omitted. BEVFormer V2 fills missing BEV features + from the nearest available frame during fusion, matching its official + dataset path. + """ + if current_frame_index < scene_start_index: + raise ValueError("current frame precedes the scene start") + if source_hz <= 0 or model_hz <= 0 or source_hz % model_hz: + raise ValueError("source_hz must be a positive multiple of model_hz") + if not model_frames or 0 not in model_frames: + raise ValueError("model_frames must include the current frame") + if len(set(model_frames)) != len(model_frames): + raise ValueError("model_frames must be unique") + if any(frame > 0 for frame in model_frames): + raise ValueError("future frames are not valid detector inputs") + + stride = source_hz // model_hz + selected = { + int(offset): current_frame_index + int(offset) * stride + for offset in sorted(model_frames) + if current_frame_index + int(offset) * stride >= scene_start_index + } + if selected.get(0) != current_frame_index: + raise ValueError("current frame selection is invalid") + return selected + + +def pose_to_world_from_top_lidar( + *, + latitude_deg: float, + longitude_deg: float, + heading_deg_cw_from_north: float, + origin_latitude_deg: float, + origin_longitude_deg: float, +) -> np.ndarray: + """Approximate a local ENU transform for one packed KITScenes pose. + + The Console snapshot carries WGS84 position and heading rather than the raw + six-degree-of-freedom pose. The small-area equirectangular conversion keeps + translation in metres; roll and pitch are intentionally unavailable. + """ + values = ( + latitude_deg, + longitude_deg, + heading_deg_cw_from_north, + origin_latitude_deg, + origin_longitude_deg, + ) + if not all(math.isfinite(value) for value in values): + raise ValueError("pose coordinates must be finite") + if not -90.0 <= latitude_deg <= 90.0: + raise ValueError("latitude is outside WGS84 bounds") + if not -180.0 <= longitude_deg <= 180.0: + raise ValueError("longitude is outside WGS84 bounds") + + earth_radius_m = 6_378_137.0 + origin_latitude_rad = math.radians(origin_latitude_deg) + east_m = ( + math.radians(longitude_deg - origin_longitude_deg) + * earth_radius_m + * math.cos(origin_latitude_rad) + ) + north_m = ( + math.radians(latitude_deg - origin_latitude_deg) + * earth_radius_m + ) + heading_rad = math.radians(heading_deg_cw_from_north) + sin_heading = math.sin(heading_rad) + cos_heading = math.cos(heading_rad) + + transform = np.eye(4, dtype=np.float64) + # Columns are the FLU forward, left, and up axes expressed in local ENU. + transform[:3, :3] = np.asarray( + [ + [sin_heading, -cos_heading, 0.0], + [cos_heading, sin_heading, 0.0], + [0.0, 0.0, 1.0], + ], + dtype=np.float64, + ) + transform[:3, 3] = [east_m, north_m, 0.0] + return transform + + +def align_history_projection_to_current( + projection_ref_to_camera: np.ndarray, + *, + history_to_world: np.ndarray, + current_to_world: np.ndarray, +) -> tuple[np.ndarray, np.ndarray]: + """Express a history camera projection in the current top-lidar frame.""" + projection = np.asarray(projection_ref_to_camera, dtype=np.float64) + history_pose = np.asarray(history_to_world, dtype=np.float64) + current_pose = np.asarray(current_to_world, dtype=np.float64) + if projection.shape != (3, 4): + raise ValueError("projection must have shape [3,4]") + if history_pose.shape != (4, 4) or current_pose.shape != (4, 4): + raise ValueError("poses must have shape [4,4]") + if not ( + np.isfinite(projection).all() + and np.isfinite(history_pose).all() + and np.isfinite(current_pose).all() + ): + raise ValueError("projection transforms must be finite") + + history_to_current = np.linalg.inv(current_pose) @ history_pose + current_to_history = np.linalg.inv(history_to_current) + return projection @ current_to_history, history_to_current + + +def scale_packed_projection( + projection_ref_to_camera: np.ndarray, + *, + packed_width: int = 256, + packed_height: int = 256, + model_width: int = 640, + model_height: int = 256, +) -> np.ndarray: + """Scale packed-image intrinsics to the BEVFormer evaluation tensor.""" + if min(packed_width, packed_height, model_width, model_height) <= 0: + raise ValueError("image dimensions must be positive") + projection = np.asarray(projection_ref_to_camera, dtype=np.float64) + if projection.shape != (3, 4) or not np.isfinite(projection).all(): + raise ValueError("projection must be a finite [3,4] matrix") + image_scale = np.diag( + [ + model_width / packed_width, + model_height / packed_height, + 1.0, + ] + ) + return image_scale @ projection + + +def rasterize_detection_boxes( + detections: Sequence[DetectionBox], + *, + geometry: NavigationRasterGeometry = AUTOE2E_NAVIGATION_GEOMETRY, + score_threshold: float = 0.2, + max_detections: int = 300, +) -> np.ndarray: + """Rasterize box footprints into `[8,H,W]` semantic probabilities. + + A cell is occupied when its physical centre lies within the exact predicted + oriented rectangle. Overlapping predictions use maximum confidence. + """ + if not math.isfinite(score_threshold) or not 0.0 <= score_threshold <= 1.0: + raise ValueError("score_threshold must be in [0,1]") + if max_detections <= 0: + raise ValueError("max_detections must be positive") + + probability = np.zeros( + ( + len(SEMANTIC_CLASS_NAMES), + geometry.height_px, + geometry.width_px, + ), + dtype=np.float32, + ) + selected = sorted( + ( + detection + for detection in detections + if detection.score >= score_threshold + ), + key=lambda detection: detection.score, + reverse=True, + )[:max_detections] + if not selected: + return probability + + x_grid, y_grid = geometry.pixel_center_grids() + for detection in selected: + semantic_name = semantic_class_for_detection(detection.class_name) + class_index = SEMANTIC_CLASS_NAMES.index(semantic_name) + cos_yaw = math.cos(detection.yaw_rad) + sin_yaw = math.sin(detection.yaw_rad) + delta_x = x_grid - detection.center_x_m + delta_y = y_grid - detection.center_y_m + # Match mmdet3d v0.17.1 LiDARInstance3DBoxes.corners. Its row-vector + # local-to-LiDAR rotation is clockwise, so this is the inverse map. + box_forward = delta_x * cos_yaw - delta_y * sin_yaw + box_left = delta_x * sin_yaw + delta_y * cos_yaw + occupied = ( + (np.abs(box_forward) <= detection.length_m * 0.5) + & (np.abs(box_left) <= detection.width_m * 0.5) + ) + probability[class_index, occupied] = np.maximum( + probability[class_index, occupied], + detection.score, + ) + return probability + + +def provenance() -> Mapping[str, object]: + """Return public scientific provenance for the dedicated Dashboard.""" + return { + "artifact_kind": BEVFORMER_V2_ARTIFACT_KIND, + "config": BEVFORMER_V2_CONFIG_NAME, + "head_version": BEVFORMER_V2_HEAD_VERSION, + "repository": BEVFORMER_V2_REPOSITORY, + "repository_revision": BEVFORMER_V2_REVISION, + "weight_sha256": BEVFORMER_V2_WEIGHT_SHA256, + "weight_source_url": BEVFORMER_V2_WEIGHT_SOURCE_URL, + "code_license_spdx": BEVFORMER_V2_CODE_LICENSE_SPDX, + "weight_license_spdx": BEVFORMER_V2_WEIGHT_LICENSE_SPDX, + "training_data_license_spdx": ( + BEVFORMER_V2_TRAINING_DATA_LICENSE_SPDX + ), + "supported_semantic_classes": list( + BEVFORMER_V2_SUPPORTED_SEMANTIC_CLASSES + ), + "teacher_available": False, + "limitations": [ + "Official BEVFormer V2 publishes 3-D detection, not BEV segmentation.", + "Object occupancy is derived only from predicted box footprints.", + "Road and map classes are unsupported and remain empty.", + ( + "KITScenes packed square images are stretched to the official " + "2.5:1 input aspect; this changes the source-camera aspect " + "ratio and uses 2.5x lower linear resolution than official " + "nuScenes evaluation." + ), + "Teacher and Error views are unavailable without perception labels.", + ( + "The public weight has no separately stated license; its " + "nuScenes training data is CC-BY-NC-SA-4.0." + ), + ], + } diff --git a/Platform/pipelines/bevformer_v2_runtime.py b/Platform/pipelines/bevformer_v2_runtime.py new file mode 100644 index 000000000..f254bc03d --- /dev/null +++ b/Platform/pipelines/bevformer_v2_runtime.py @@ -0,0 +1,527 @@ +"""Official BEVFormer V2 inference over packed KITScenes samples. + +OpenMMLab and BEVFormer imports stay behind the model-loading boundary so the +packing, geometry, and rasterization contracts remain unit-testable without +the legacy CUDA runtime. +""" + +from __future__ import annotations + +import hashlib +import importlib +import io +import json +import sys +import tarfile +from collections import OrderedDict +from collections.abc import Iterable, Mapping, MutableMapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np + +from data_processing.geospatial import decode_pose +from Platform.pipelines.bevformer_v2_occupancy import ( + BEVFORMER_V2_CONFIG_NAME, + BEVFORMER_V2_FRAMES, + BEVFORMER_V2_REVISION, + BEVFORMER_V2_WEIGHT_SHA256, + DetectionBox, + align_history_projection_to_current, + pose_to_world_from_top_lidar, + rasterize_detection_boxes, + scale_packed_projection, + temporal_frame_indices, +) + +BEVFORMER_V2_CAMERA_COUNT = 6 +# KITScenes stores front-left before front-right. The official nuScenes info +# converter and learned camera embeddings use front-right before front-left. +BEVFORMER_V2_CAMERA_ORDER = (0, 2, 1, 3, 4, 5) +BEVFORMER_V2_IMAGE_HEIGHT = 256 +BEVFORMER_V2_IMAGE_WIDTH = 640 +BEVFORMER_V2_IMAGE_MEAN_BGR = (103.53, 116.28, 123.675) +BEVFORMER_V2_CLASS_NAMES = ( + "barrier", + "bicycle", + "bus", + "car", + "construction_vehicle", + "motorcycle", + "pedestrian", + "traffic_cone", + "trailer", + "truck", +) + + +@dataclass(frozen=True) +class PackedBEVFormerFrame: + """One packed KITScenes frame before detector-specific preprocessing.""" + + sample_uid: str + episode_id: str + frame_index: int + timestamp_ns: int + image_payloads: tuple[bytes, ...] + projection_ref_to_camera: np.ndarray + pose: Mapping[str, float | int] + + def __post_init__(self) -> None: + if not self.sample_uid or not self.episode_id: + raise ValueError("packed frame identity must not be empty") + if self.frame_index < 0: + raise ValueError("packed frame index must be non-negative") + if len(self.image_payloads) != BEVFORMER_V2_CAMERA_COUNT: + raise ValueError("BEVFormer V2 requires six camera payloads") + if any(not payload for payload in self.image_payloads): + raise ValueError("packed camera payloads must not be empty") + projection = np.asarray(self.projection_ref_to_camera) + if projection.shape != (BEVFORMER_V2_CAMERA_COUNT, 3, 4): + raise ValueError("packed projection must have shape [6,3,4]") + if not np.isfinite(projection).all(): + raise ValueError("packed projection must be finite") + + +def _sample_from_members( + sample_uid: str, + members: Mapping[str, bytes], +) -> PackedBEVFormerFrame: + required = {"meta.json", "calib.json", "pose.npy"} + required.update( + f"cam_{camera}.jpg" + for camera in range(BEVFORMER_V2_CAMERA_COUNT) + ) + missing = required - set(members) + if missing: + raise ValueError( + f"packed sample {sample_uid!r} is missing {sorted(missing)}" + ) + metadata = json.loads(members["meta.json"]) + calibration = json.loads(members["calib.json"]) + if not isinstance(metadata, Mapping) or not isinstance( + calibration, + Mapping, + ): + raise ValueError("packed metadata must be JSON objects") + if metadata.get("sample_uid") not in (None, sample_uid): + raise ValueError("packed sample UID differs from its tar key") + projection_spec = calibration.get("projection") + if ( + not isinstance(projection_spec, Mapping) + or projection_spec.get("type") != "pinhole" + or projection_spec.get("reference_frame") != "top_lidar_flu" + ): + raise ValueError( + "BEVFormer V2 requires top_lidar_flu pinhole calibration" + ) + pose = decode_pose(members["pose.npy"]) + timestamp_ns = int(pose["timestamp_ns"]) + frame_index = metadata.get("frame_idx") + if isinstance(frame_index, bool) or not isinstance(frame_index, int): + raise ValueError("packed frame_idx must be an integer") + episode_id = metadata.get("split_group_uid") + if not isinstance(episode_id, str) or not episode_id: + raise ValueError("packed sample has no scene identity") + projection = np.asarray( + projection_spec.get("matrix"), + dtype=np.float64, + ) + projection = projection[np.asarray(BEVFORMER_V2_CAMERA_ORDER)] + return PackedBEVFormerFrame( + sample_uid=sample_uid, + episode_id=episode_id, + frame_index=frame_index, + timestamp_ns=timestamp_ns, + image_payloads=tuple( + members[f"cam_{camera}.jpg"] + for camera in BEVFORMER_V2_CAMERA_ORDER + ), + projection_ref_to_camera=projection, + pose=pose, + ) + + +def iter_packed_bevformer_frames( + tar_path: str | Path, +) -> Iterable[PackedBEVFormerFrame]: + """Yield contiguous WebDataset samples without decoding unrelated members.""" + current_key: str | None = None + current_members: dict[str, bytes] = {} + with tarfile.open(tar_path, mode="r:*") as archive: + for member in archive: + if not member.isfile() or "." not in member.name: + continue + sample_uid, suffix = member.name.split(".", 1) + if current_key is not None and sample_uid != current_key: + yield _sample_from_members(current_key, current_members) + current_members = {} + current_key = sample_uid + if suffix not in { + "meta.json", + "calib.json", + "pose.npy", + *( + f"cam_{camera}.jpg" + for camera in range(BEVFORMER_V2_CAMERA_COUNT) + ), + }: + continue + stream = archive.extractfile(member) + if stream is None: + raise ValueError(f"could not read tar member {member.name!r}") + current_members[suffix] = stream.read() + if current_key is not None: + yield _sample_from_members(current_key, current_members) + + +def temporal_frames_for( + current: PackedBEVFormerFrame, + history: Mapping[int, PackedBEVFormerFrame], +) -> OrderedDict[int, PackedBEVFormerFrame]: + """Select exact 2 Hz history, omitting unavailable early packed frames.""" + selected = temporal_frame_indices(current.frame_index) + frames: OrderedDict[int, PackedBEVFormerFrame] = OrderedDict() + for offset, frame_index in selected.items(): + frame = current if offset == 0 else history.get(frame_index) + if frame is None: + continue + if frame.episode_id != current.episode_id: + raise ValueError("temporal history crossed a KITScenes scene") + frames[offset] = frame + if frames.get(0) is not current: + raise ValueError("temporal selection omitted the current frame") + return frames + + +def _world_pose( + frame: PackedBEVFormerFrame, + *, + origin_latitude_deg: float, + origin_longitude_deg: float, +) -> np.ndarray: + return pose_to_world_from_top_lidar( + latitude_deg=float(frame.pose["latitude_deg"]), + longitude_deg=float(frame.pose["longitude_deg"]), + heading_deg_cw_from_north=float( + frame.pose["heading_deg_cw_from_north"] + ), + origin_latitude_deg=origin_latitude_deg, + origin_longitude_deg=origin_longitude_deg, + ) + + +def _packed_image_size(payload: bytes) -> tuple[int, int]: + from PIL import Image + + with Image.open(io.BytesIO(payload)) as source: + source.verify() + return source.size + + +def bevformer_metadata_for( + frames: Mapping[int, PackedBEVFormerFrame], + *, + box_type_3d: Any, +) -> OrderedDict[int, dict[str, Any]]: + """Build the official V2 metadata map in current top-lidar coordinates.""" + current = frames.get(0) + if current is None: + raise ValueError("BEVFormer metadata requires offset zero") + origin_latitude = float(current.pose["latitude_deg"]) + origin_longitude = float(current.pose["longitude_deg"]) + current_to_world = _world_pose( + current, + origin_latitude_deg=origin_latitude, + origin_longitude_deg=origin_longitude, + ) + output: OrderedDict[int, dict[str, Any]] = OrderedDict() + for offset, frame in sorted(frames.items()): + frame_to_world = _world_pose( + frame, + origin_latitude_deg=origin_latitude, + origin_longitude_deg=origin_longitude, + ) + projections = [] + frame_to_current: np.ndarray | None = None + packed_sizes = tuple( + _packed_image_size(payload) + for payload in frame.image_payloads + ) + if len(set(packed_sizes)) != 1: + raise ValueError( + "BEVFormer V2 requires equal packed camera dimensions" + ) + packed_width, packed_height = packed_sizes[0] + for packed_projection in frame.projection_ref_to_camera: + if offset == 0: + aligned = packed_projection + else: + aligned, frame_to_current = ( + align_history_projection_to_current( + packed_projection, + history_to_world=frame_to_world, + current_to_world=current_to_world, + ) + ) + scaled = scale_packed_projection( + aligned, + packed_width=packed_width, + packed_height=packed_height, + ) + homogeneous = np.eye(4, dtype=np.float32) + homogeneous[:3, :] = scaled.astype(np.float32) + projections.append(homogeneous) + image_shape = ( + BEVFORMER_V2_IMAGE_HEIGHT, + BEVFORMER_V2_IMAGE_WIDTH, + 3, + ) + output[offset] = { + "box_mode_3d": None, + "box_type_3d": box_type_3d, + "filename": [frame.sample_uid] * BEVFORMER_V2_CAMERA_COUNT, + "flip": False, + "img_norm_cfg": { + "mean": np.asarray( + BEVFORMER_V2_IMAGE_MEAN_BGR, + dtype=np.float32, + ), + "std": np.ones(3, dtype=np.float32), + "to_rgb": False, + }, + "img_shape": [image_shape] * BEVFORMER_V2_CAMERA_COUNT, + "lidar2img": projections, + "lidaradj2lidarcurr": frame_to_current, + "ori_shape": [image_shape] * BEVFORMER_V2_CAMERA_COUNT, + "pad_shape": [image_shape] * BEVFORMER_V2_CAMERA_COUNT, + "sample_idx": frame.sample_uid, + "scale_factor": np.asarray( + [ + BEVFORMER_V2_IMAGE_WIDTH / packed_width, + BEVFORMER_V2_IMAGE_HEIGHT / packed_height, + BEVFORMER_V2_IMAGE_WIDTH / packed_width, + BEVFORMER_V2_IMAGE_HEIGHT / packed_height, + ], + dtype=np.float32, + ), + "scene_token": frame.episode_id, + "timestamp": frame.timestamp_ns / 1_000_000_000.0, + } + return output + + +def preprocess_packed_images( + frame: PackedBEVFormerFrame, +) -> Any: + """Return `[6,3,256,640]` BGR float images for the official backbone.""" + import torch + from PIL import Image + + images = [] + mean = np.asarray( + BEVFORMER_V2_IMAGE_MEAN_BGR, + dtype=np.float32, + ) + for payload in frame.image_payloads: + with Image.open(io.BytesIO(payload)) as source: + rgb = source.convert("RGB").resize( + ( + BEVFORMER_V2_IMAGE_WIDTH, + BEVFORMER_V2_IMAGE_HEIGHT, + ), + resample=Image.Resampling.BILINEAR, + ) + values = np.asarray(rgb, dtype=np.float32) + bgr = np.ascontiguousarray(values[:, :, ::-1]) + bgr -= mean + images.append(torch.from_numpy(bgr).permute(2, 0, 1)) + return torch.stack(images) + + +def bevformer_batch_for( + frames: Mapping[int, PackedBEVFormerFrame], + *, + box_type_3d: Any, + device: Any, +) -> tuple[Any, list[list[OrderedDict[int, dict[str, Any]]]]]: + """Build the exact image and metadata nesting expected by forward_test.""" + import torch + + ordered = OrderedDict(sorted(frames.items())) + images = torch.stack( + [preprocess_packed_images(frame) for frame in ordered.values()] + ).unsqueeze(0) + images = images.to(device=device, non_blocking=True) + metadata = bevformer_metadata_for( + ordered, + box_type_3d=box_type_3d, + ) + return images, [[metadata]] + + +def detections_from_bevformer_result( + result: Any, +) -> list[DetectionBox]: + """Convert one official mmdet3d result without changing box geometry.""" + if ( + not isinstance(result, Sequence) + or len(result) != 1 + or not isinstance(result[0], Mapping) + or not isinstance(result[0].get("pts_bbox"), Mapping) + ): + raise ValueError("BEVFormer result has an unexpected envelope") + boxes = result[0]["pts_bbox"] + box_tensor = boxes.get("boxes_3d") + scores = boxes.get("scores_3d") + labels = boxes.get("labels_3d") + if box_tensor is None or scores is None or labels is None: + raise ValueError("BEVFormer result is missing 3-D boxes") + tensor = np.asarray(box_tensor.tensor.detach().cpu(), dtype=np.float64) + score_values = np.asarray(scores.detach().cpu(), dtype=np.float64) + label_values = np.asarray(labels.detach().cpu(), dtype=np.int64) + if ( + tensor.ndim != 2 + or tensor.shape[1] < 7 + or score_values.shape != (tensor.shape[0],) + or label_values.shape != (tensor.shape[0],) + ): + raise ValueError("BEVFormer result tensor shapes are invalid") + detections = [] + for box, score, label in zip(tensor, score_values, label_values): + if label < 0 or label >= len(BEVFORMER_V2_CLASS_NAMES): + raise ValueError("BEVFormer returned an unknown class index") + detections.append( + DetectionBox( + class_name=BEVFORMER_V2_CLASS_NAMES[int(label)], + score=float(score), + center_x_m=float(box[0]), + center_y_m=float(box[1]), + length_m=float(box[3]), + width_m=float(box[4]), + yaw_rad=float(box[6]), + ) + ) + return detections + + +def infer_bevformer_frame( + model: Any, + current: PackedBEVFormerFrame, + history: Mapping[int, PackedBEVFormerFrame], + *, + box_type_3d: Any, + device: Any, + score_threshold: float = 0.2, +) -> np.ndarray: + """Run official inference and return one uncorrected `[8,450,300]` raster.""" + import torch + + frames = temporal_frames_for(current, history) + images, metadata = bevformer_batch_for( + frames, + box_type_3d=box_type_3d, + device=device, + ) + with torch.no_grad(): + result = model( + return_loss=False, + img=[images], + img_metas=metadata, + rescale=True, + ) + return rasterize_detection_boxes( + detections_from_bevformer_result(result), + score_threshold=score_threshold, + ) + + +def remember_history_frame( + history: MutableMapping[int, PackedBEVFormerFrame], + frame: PackedBEVFormerFrame, +) -> None: + """Retain only the exact 3.5 second history needed by the t8 model.""" + stale_before = frame.frame_index + min(BEVFORMER_V2_FRAMES) * 5 + for frame_index in list(history): + if frame_index < stale_before: + del history[frame_index] + history[frame.frame_index] = frame + + +def sha256_file(path: str | Path, chunk_size: int = 8 << 20) -> str: + digest = hashlib.sha256() + with Path(path).open("rb") as stream: + while chunk := stream.read(chunk_size): + digest.update(chunk) + return digest.hexdigest() + + +def load_official_bevformer_v2( + *, + repository_path: str | Path, + checkpoint_path: str | Path, + device: Any, +) -> tuple[Any, Any]: + """Load only the pinned Apache-2.0 BEVFormer revision and checkpoint.""" + repository = Path(repository_path).resolve() + checkpoint = Path(checkpoint_path).resolve() + if sha256_file(checkpoint) != BEVFORMER_V2_WEIGHT_SHA256: + raise ValueError("BEVFormer V2 weight digest does not match provenance") + git_head = repository / ".git" / "HEAD" + if not git_head.exists(): + raise ValueError("BEVFormer repository has no revision metadata") + import subprocess + + revision = subprocess.run( + ["git", "-C", str(repository), "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + if revision != BEVFORMER_V2_REVISION: + raise ValueError("BEVFormer repository revision is not pinned") + + sys.path.insert(0, str(repository)) + try: + from mmcv import Config + from mmcv.runner import load_checkpoint + from mmdet3d.core.bbox import LiDARInstance3DBoxes + from mmdet3d.models import build_model + + importlib.import_module("projects.mmdet3d_plugin") + config_path = ( + repository + / "projects" + / "configs" + / "bevformerv2" + / BEVFORMER_V2_CONFIG_NAME + ) + config = Config.fromfile(str(config_path)) + config.model.pretrained = None + config.model.train_cfg = None + model = build_model( + config.model, + test_cfg=config.get("test_cfg"), + ) + loaded = load_checkpoint( + model, + str(checkpoint), + map_location="cpu", + ) + metadata = loaded.get("meta", {}) if isinstance(loaded, Mapping) else {} + model.CLASSES = metadata.get( + "CLASSES", + BEVFORMER_V2_CLASS_NAMES, + ) + if tuple(model.CLASSES) != BEVFORMER_V2_CLASS_NAMES: + raise ValueError( + "BEVFormer checkpoint class order differs from the pinned " + "detector contract" + ) + model.to(device) + model.eval() + return model, LiDARInstance3DBoxes + finally: + if sys.path and sys.path[0] == str(repository): + sys.path.pop(0) diff --git a/Platform/pipelines/distributed_training.py b/Platform/pipelines/distributed_training.py new file mode 100644 index 000000000..83a221286 --- /dev/null +++ b/Platform/pipelines/distributed_training.py @@ -0,0 +1,705 @@ +"""Flyte Ray tasks for distributed AutoE2E training.""" + +from __future__ import annotations + +import json +import os +import re +from pathlib import Path +from typing import List, NamedTuple, Optional + +from flytekit import ( + PodTemplate, + Resources, + current_context, + task, + workflow, +) +from flytekit.types.directory import FlyteDirectory +from flytekit.types.file import FlyteFile +from flytekitplugins.ray import ( + HeadNodeConfig, + RayJobConfig, + WorkerNodeConfig, +) +from kubernetes.client import ( + V1Affinity, + V1Container, + V1EmptyDirVolumeSource, + V1EnvVar, + V1LabelSelector, + V1LabelSelectorRequirement, + V1PodAffinityTerm, + V1PodAntiAffinity, + V1PodSpec, + V1ResourceRequirements, + V1Volume, + V1VolumeMount, +) + + +TRAINING_IMAGE = os.environ.get( + "AUTO_E2E_TRAINING_IMAGE", + "auto-e2e/training:latest", +) +RAY_STORAGE_PATH = os.environ.get( + "AUTO_E2E_RAY_STORAGE_PATH", + "s3://auto-e2e-platform-checkpoints/ray-train", +) +RAY_TASK_ENVIRONMENT = { + "AWS_DEFAULT_REGION": "us-west-2", + "AUTO_E2E_RAY_STORAGE_PATH": RAY_STORAGE_PATH, + "RAY_TRAIN_V2_ENABLED": "1", +} + + +class RaySmokeOutput(NamedTuple): + report: FlyteFile + + +class ReactiveRayOutput(NamedTuple): + checkpoint: FlyteFile + metadata: FlyteFile + checkpoint_uri: str + checkpoint_sha256: str + + +class ReactiveDistributedProgramOutput(NamedTuple): + stage_a_checkpoint: FlyteFile + stage_a_metadata: FlyteFile + stage_a_checkpoint_uri: str + stage_a_checkpoint_sha256: str + stage_b_checkpoint: FlyteFile + stage_b_metadata: FlyteFile + stage_b_checkpoint_uri: str + stage_b_checkpoint_sha256: str + + +class ReactiveCanaryOutput(NamedTuple): + stage_a_checkpoint: FlyteFile + stage_b_checkpoint: FlyteFile + stage_a_metadata: FlyteFile + stage_b_metadata: FlyteFile + gate_report: FlyteFile + + +def _head_pod_template() -> PodTemplate: + return PodTemplate( + primary_container_name="ray-head", + labels={"auto-e2e.training/role": "ray-head"}, + annotations={"karpenter.sh/do-not-disrupt": "true"}, + pod_spec=V1PodSpec( + service_account_name="default", + containers=[ + V1Container( + name="ray-head", + resources=V1ResourceRequirements( + requests={"cpu": "2", "memory": "16Gi"}, + limits={"cpu": "2", "memory": "16Gi"}, + ), + volume_mounts=[ + V1VolumeMount( + name="dshm", + mount_path="/dev/shm", + ), + ], + ), + ], + volumes=[ + V1Volume( + name="dshm", + empty_dir=V1EmptyDirVolumeSource( + medium="Memory", + size_limit="8Gi", + ), + ), + ], + ), + ) + + +def _worker_pod_template() -> PodTemplate: + return PodTemplate( + primary_container_name="ray-worker", + labels={"auto-e2e.training/role": "ray-gpu-worker"}, + annotations={"karpenter.sh/do-not-disrupt": "true"}, + pod_spec=V1PodSpec( + service_account_name="default", + affinity=V1Affinity( + pod_anti_affinity=V1PodAntiAffinity( + required_during_scheduling_ignored_during_execution=[ + V1PodAffinityTerm( + label_selector=V1LabelSelector( + match_expressions=[ + V1LabelSelectorRequirement( + key="auto-e2e.training/role", + operator="In", + values=["ray-gpu-worker"], + ), + ], + ), + topology_key="kubernetes.io/hostname", + ), + ], + ), + ), + containers=[ + V1Container( + name="ray-worker", + env=[ + V1EnvVar(name="NCCL_DEBUG", value="INFO"), + V1EnvVar( + name="TORCH_DISTRIBUTED_DEBUG", + value="DETAIL", + ), + ], + resources=V1ResourceRequirements( + requests={ + "cpu": "4", + "memory": "16Gi", + "nvidia.com/gpu": "1", + }, + limits={ + "cpu": "4", + "memory": "16Gi", + "nvidia.com/gpu": "1", + }, + ), + volume_mounts=[ + V1VolumeMount( + name="dshm", + mount_path="/dev/shm", + ), + ], + ), + ], + volumes=[ + V1Volume( + name="dshm", + empty_dir=V1EmptyDirVolumeSource( + medium="Memory", + size_limit="8Gi", + ), + ), + ], + ), + ) + + +def _ray_job_config(replicas: int) -> RayJobConfig: + return RayJobConfig( + head_node_config=HeadNodeConfig( + ray_start_params={ + "dashboard-host": "0.0.0.0", + "num-cpus": "0", + }, + pod_template=_head_pod_template(), + ), + worker_node_config=[ + WorkerNodeConfig( + group_name="gpu-workers", + replicas=replicas, + min_replicas=replicas, + max_replicas=replicas, + ray_start_params={ + "num-cpus": "4", + "num-gpus": "1", + }, + pod_template=_worker_pod_template(), + ), + ], + enable_autoscaling=False, + address="auto", + shutdown_after_job_finishes=True, + ttl_seconds_after_finished=300, + ) + + +RAY_2 = _ray_job_config(2) +RAY_4 = _ray_job_config(4) +RAY_8 = _ray_job_config(8) + + +@task( + task_config=RAY_4, + container_image=TRAINING_IMAGE, + retries=1, + labels={ + "kueue.x-k8s.io/queue-name": "training", + "kueue.x-k8s.io/priority-class": "research-low", + }, + environment=RAY_TASK_ENVIRONMENT, +) +def ray_ddp_smoke_4(steps: int = 4) -> RaySmokeOutput: + from distributed_training.ray_smoke import run_smoke + + context = current_context() + execution_name = ( + context.execution_id.name + if context.execution_id is not None + else "local" + ) + run_name = re.sub( + r"[^a-zA-Z0-9_-]", + "-", + f"{execution_name}-ray-ddp-smoke-4", + ) + result = run_smoke( + num_workers=4, + steps=steps, + storage_path=RAY_STORAGE_PATH, + run_name=run_name, + ) + report_path = Path("/tmp/ray-ddp-smoke/report.json") + report_path.parent.mkdir(parents=True, exist_ok=True) + report_path.write_text( + json.dumps(result, indent=2, sort_keys=True) + "\n", + encoding="ascii", + ) + return RaySmokeOutput(report=FlyteFile(str(report_path))) + + +def _flyte_remote_uri(value: FlyteDirectory | FlyteFile) -> str: + remote_source = str(getattr(value, "remote_source", "") or "") + uri = remote_source or str(value) + if not uri.startswith("s3://"): + raise ValueError( + "distributed Ray workers require immutable S3 inputs" + ) + return uri.rstrip("/") + + +def _run_reactive_stage_task( + *, + shards: List[FlyteDirectory], + stage: str, + num_workers: int, + parent_checkpoint: Optional[FlyteFile], + backbone: str, + epochs: int, + learning_rate: float, + weight_decay: float, + grad_clip: float, + val_fraction: float, + num_loader_workers: int, + training_seed: int, + precision: str, + gradient_accumulation_steps: int, + steps_per_epoch: int, + shuffle_buffer: int, + is_pretrained: bool, + bev_weight: float, + route_weight: float, + bev_pos_weights: List[float], + corridor_pos_weight: float, +) -> ReactiveRayOutput: + from distributed_training.reactive_stage import run_reactive_stage + + context = current_context() + execution_name = ( + context.execution_id.name + if context.execution_id is not None + else "local" + ) + run_name = re.sub( + r"[^a-zA-Z0-9_-]", + "-", + f"{execution_name}-{stage}-ray-{num_workers}", + ) + source_uris = [_flyte_remote_uri(shard) for shard in shards] + parent_uri = ( + _flyte_remote_uri(parent_checkpoint) + if parent_checkpoint is not None + else "" + ) + result = run_reactive_stage({ + "backbone": backbone, + "bev_pos_weights": list(bev_pos_weights), + "bev_weight": bev_weight, + "corridor_pos_weight": corridor_pos_weight, + "epochs": epochs, + "grad_clip": grad_clip, + "gradient_accumulation_steps": ( + gradient_accumulation_steps + ), + "is_pretrained": is_pretrained, + "learning_rate": learning_rate, + "local_cache_root": "/tmp/auto-e2e-reactive", + "num_loader_workers": num_loader_workers, + "num_workers": num_workers, + "parent_checkpoint_uri": parent_uri, + "per_rank_batch_size": 1, + "precision": precision, + "route_weight": route_weight, + "run_name": run_name, + "shuffle_buffer": shuffle_buffer, + "source_uris": source_uris, + "stage": stage, + "steps_per_epoch": steps_per_epoch, + "storage_path": RAY_STORAGE_PATH, + "training_seed": training_seed, + "use_gpu": True, + "val_fraction": val_fraction, + "weight_decay": weight_decay, + }) + metadata_path = ( + Path("/tmp/reactive-ray") + / run_name + / "metadata.json" + ) + metadata_path.parent.mkdir(parents=True, exist_ok=True) + metadata_path.write_text( + json.dumps(result, indent=2, sort_keys=True) + "\n", + encoding="ascii", + ) + metrics = result["metrics"] + return ReactiveRayOutput( + checkpoint=FlyteFile(result["checkpoint_file_uri"]), + metadata=FlyteFile(str(metadata_path)), + checkpoint_uri=str(result["checkpoint_file_uri"]), + checkpoint_sha256=str(metrics["checkpoint_sha256"]), + ) + + +@task( + container_image=TRAINING_IMAGE, + requests=Resources(cpu="2", mem="8Gi"), + limits=Resources(cpu="2", mem="8Gi"), +) +def build_reactive_canary_dataset(stage: str) -> FlyteDirectory: + """Create deterministic production-schema shards for the GPU gate.""" + import tempfile + from pathlib import Path + + from distributed_training.reactive_canary_data import ( + write_reactive_canary_dataset, + ) + from training.reactive_multitask import ReactiveTrainingStage + + training_stage = ReactiveTrainingStage(stage) + output = Path(tempfile.mkdtemp(prefix=f"reactive-{stage}-")) + write_reactive_canary_dataset( + output, + stage=training_stage, + shard_count=2, + train_samples_per_shard=2, + validation_samples_per_shard=1, + ) + return FlyteDirectory(str(output)) + + +@task( + container_image=TRAINING_IMAGE, + requests=Resources(cpu="1", mem="2Gi"), + limits=Resources(cpu="1", mem="2Gi"), +) +def verify_reactive_canary_training( + stage_a_metadata: FlyteFile, + stage_b_metadata: FlyteFile, +) -> FlyteFile: + """Fail when the real-model two-stage GPU canary is not learning.""" + import math + import tempfile + from pathlib import Path + + reports = {} + for stage_name, source in ( + ("stage_a", stage_a_metadata), + ("stage_b", stage_b_metadata), + ): + payload = json.loads(Path(source.download()).read_text()) + history = payload.get("history") + if not isinstance(history, list) or len(history) < 2: + raise ValueError( + f"{stage_name} canary needs at least two reported epochs" + ) + required = ( + "train_bev_segmentation", + "train_route_reconstruction", + "train_total", + "train_trajectory", + "validation_ade_6p4s_m", + ) + for epoch in history: + if any( + name not in epoch + or not math.isfinite(float(epoch[name])) + for name in required + ): + raise ValueError( + f"{stage_name} canary emitted non-finite metrics" + ) + reports[stage_name] = history + + stage_a = reports["stage_a"] + stage_b = reports["stage_b"] + if float(stage_a[0]["train_bev_segmentation"]) <= 0.0: + raise ValueError("Stage A canary did not execute the BEV loss") + if any( + abs(float(epoch["train_bev_segmentation"])) > 1e-12 + for epoch in stage_b + ): + raise ValueError("Stage B canary executed the BEV loss") + initial_total = float(stage_a[0]["train_total"]) + minimum_later_total = min( + float(epoch["train_total"]) for epoch in stage_a[1:] + ) + if minimum_later_total >= initial_total: + raise ValueError( + "Stage A canary total loss did not decrease: " + f"initial={initial_total} later_min={minimum_later_total}" + ) + + report = { + "schema_version": "reactive_ddp_canary_report_v1", + "stage_a_initial_total": initial_total, + "stage_a_minimum_later_total": minimum_later_total, + "stage_a_epochs": len(stage_a), + "stage_b_epochs": len(stage_b), + "stage_b_bev_loss_disabled": True, + "thresholds_pass": True, + } + output = ( + Path(tempfile.mkdtemp(prefix="reactive-canary-report-")) + / "report.json" + ) + output.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", + encoding="ascii", + ) + return FlyteFile(str(output)) + + +@task( + task_config=RAY_2, + container_image=TRAINING_IMAGE, + retries=1, + labels={ + "kueue.x-k8s.io/queue-name": "training", + "kueue.x-k8s.io/priority-class": "research-low", + }, + environment=RAY_TASK_ENVIRONMENT, +) +def train_reactive_stage_ray_2( + shards: List[FlyteDirectory], + stage: str, + parent_checkpoint: Optional[FlyteFile] = None, + backbone: str = "swin_v2_tiny", + epochs: int = 2, + learning_rate: float = 1e-4, + weight_decay: float = 1e-2, + grad_clip: float = 1.0, + val_fraction: float = 0.1, + num_loader_workers: int = 2, + training_seed: int = 149, + precision: str = "fp32", + gradient_accumulation_steps: int = 1, + steps_per_epoch: int = 2, + shuffle_buffer: int = 64, + is_pretrained: bool = False, + bev_weight: float = 1.0, + route_weight: float = 1.0, + bev_pos_weights: Optional[List[float]] = None, + corridor_pos_weight: float = 1.0, +) -> ReactiveRayOutput: + """Run a two-node real-model integration canary.""" + return _run_reactive_stage_task( + shards=shards, + stage=stage, + num_workers=2, + parent_checkpoint=parent_checkpoint, + backbone=backbone, + epochs=epochs, + learning_rate=learning_rate, + weight_decay=weight_decay, + grad_clip=grad_clip, + val_fraction=val_fraction, + num_loader_workers=num_loader_workers, + training_seed=training_seed, + precision=precision, + gradient_accumulation_steps=gradient_accumulation_steps, + steps_per_epoch=steps_per_epoch, + shuffle_buffer=shuffle_buffer, + is_pretrained=is_pretrained, + bev_weight=bev_weight, + route_weight=route_weight, + bev_pos_weights=( + bev_pos_weights + if bev_pos_weights is not None + else [1.0] * 8 + ), + corridor_pos_weight=corridor_pos_weight, + ) + + +@task( + task_config=RAY_8, + container_image=TRAINING_IMAGE, + retries=1, + labels={ + "kueue.x-k8s.io/queue-name": "training", + "kueue.x-k8s.io/priority-class": "research-low", + }, + environment=RAY_TASK_ENVIRONMENT, +) +def train_reactive_stage_ray_8( + shards: List[FlyteDirectory], + stage: str, + parent_checkpoint: Optional[FlyteFile] = None, + backbone: str = "swin_v2_tiny", + epochs: int = 3, + learning_rate: float = 1e-4, + weight_decay: float = 1e-2, + grad_clip: float = 1.0, + val_fraction: float = 0.1, + num_loader_workers: int = 2, + training_seed: int = 149, + precision: str = "bf16", + gradient_accumulation_steps: int = 1, + steps_per_epoch: int = 0, + shuffle_buffer: int = 1000, + is_pretrained: bool = True, + bev_weight: float = 1.0, + route_weight: float = 1.0, + bev_pos_weights: Optional[List[float]] = None, + corridor_pos_weight: float = 1.0, +) -> ReactiveRayOutput: + """Run one production-size Reactive DDP stage.""" + return _run_reactive_stage_task( + shards=shards, + stage=stage, + num_workers=8, + parent_checkpoint=parent_checkpoint, + backbone=backbone, + epochs=epochs, + learning_rate=learning_rate, + weight_decay=weight_decay, + grad_clip=grad_clip, + val_fraction=val_fraction, + num_loader_workers=num_loader_workers, + training_seed=training_seed, + precision=precision, + gradient_accumulation_steps=gradient_accumulation_steps, + steps_per_epoch=steps_per_epoch, + shuffle_buffer=shuffle_buffer, + is_pretrained=is_pretrained, + bev_weight=bev_weight, + route_weight=route_weight, + bev_pos_weights=( + bev_pos_weights + if bev_pos_weights is not None + else [1.0] * 8 + ), + corridor_pos_weight=corridor_pos_weight, + ) + + +@workflow +def wf_ray_ddp_smoke_4(steps: int = 4) -> FlyteFile: + return ray_ddp_smoke_4(steps=steps).report + + +@workflow +def wf_train_reactive_nuplan_l2d_ray_8( + nuplan_shards: List[FlyteDirectory], + l2d_shards: List[FlyteDirectory], + stage_a_epochs: int = 3, + stage_b_epochs: int = 3, + stage_a_learning_rate: float = 1e-4, + stage_b_learning_rate: float = 3e-5, + val_fraction: float = 0.1, + num_loader_workers: int = 2, + training_seed: int = 149, + precision: str = "bf16", + bev_weight: float = 1.0, + route_weight: float = 1.0, +) -> ReactiveDistributedProgramOutput: + """Train Stage A and Stage B as separate eight-rank RayJobs.""" + stage_a = train_reactive_stage_ray_8( + shards=nuplan_shards, + stage="nuplan_full", + parent_checkpoint=None, + epochs=stage_a_epochs, + learning_rate=stage_a_learning_rate, + val_fraction=val_fraction, + num_loader_workers=num_loader_workers, + training_seed=training_seed, + precision=precision, + bev_weight=bev_weight, + route_weight=route_weight, + ) + stage_b = train_reactive_stage_ray_8( + shards=l2d_shards, + stage="l2d_continuation", + parent_checkpoint=stage_a.checkpoint, + epochs=stage_b_epochs, + learning_rate=stage_b_learning_rate, + val_fraction=val_fraction, + num_loader_workers=num_loader_workers, + training_seed=training_seed, + precision=precision, + bev_weight=0.0, + route_weight=route_weight, + ) + return ReactiveDistributedProgramOutput( + stage_a_checkpoint=stage_a.checkpoint, + stage_a_metadata=stage_a.metadata, + stage_a_checkpoint_uri=stage_a.checkpoint_uri, + stage_a_checkpoint_sha256=stage_a.checkpoint_sha256, + stage_b_checkpoint=stage_b.checkpoint, + stage_b_metadata=stage_b.metadata, + stage_b_checkpoint_uri=stage_b.checkpoint_uri, + stage_b_checkpoint_sha256=stage_b.checkpoint_sha256, + ) + + +@workflow +def wf_reactive_multistage_ray_2_canary() -> ReactiveCanaryOutput: + """Run the real Reactive objectives through two multi-node RayJobs.""" + stage_a_data = build_reactive_canary_dataset( + stage="nuplan_full" + ) + stage_b_data = build_reactive_canary_dataset( + stage="l2d_continuation" + ) + stage_a = train_reactive_stage_ray_2( + shards=[stage_a_data], + stage="nuplan_full", + parent_checkpoint=None, + epochs=3, + learning_rate=3e-4, + val_fraction=0.5, + num_loader_workers=1, + precision="fp32", + steps_per_epoch=4, + shuffle_buffer=0, + is_pretrained=False, + bev_weight=0.1, + route_weight=0.1, + ) + stage_b = train_reactive_stage_ray_2( + shards=[stage_b_data], + stage="l2d_continuation", + parent_checkpoint=stage_a.checkpoint, + epochs=2, + learning_rate=1e-4, + val_fraction=0.5, + num_loader_workers=1, + precision="fp32", + steps_per_epoch=2, + shuffle_buffer=0, + is_pretrained=False, + bev_weight=0.0, + route_weight=0.1, + ) + gate_report = verify_reactive_canary_training( + stage_a_metadata=stage_a.metadata, + stage_b_metadata=stage_b.metadata, + ) + return ReactiveCanaryOutput( + stage_a_checkpoint=stage_a.checkpoint, + stage_b_checkpoint=stage_b.checkpoint, + stage_a_metadata=stage_a.metadata, + stage_b_metadata=stage_b.metadata, + gate_report=gate_report, + ) diff --git a/Platform/pipelines/nuplan_acquisition.py b/Platform/pipelines/nuplan_acquisition.py new file mode 100644 index 000000000..89bc77855 --- /dev/null +++ b/Platform/pipelines/nuplan_acquisition.py @@ -0,0 +1,743 @@ +"""Pure contracts for one-time authorized nuPlan dataset acquisition.""" + +from __future__ import annotations + +import hashlib +import ipaddress +import json +import re +import socket +from collections.abc import Mapping, Sequence +from typing import Any, BinaryIO +from urllib.parse import urlsplit + + +SOURCE_SCHEMA_VERSION = "nuplan_authorized_source_v1" +ARCHIVE_RECEIPT_SCHEMA_VERSION = "nuplan_archive_receipt_v1" +SNAPSHOT_SCHEMA_VERSION = "nuplan_raw_snapshot_v1" +REQUIRED_COMPONENTS = frozenset({"maps", "database", "sensor_blobs"}) +MIN_MULTIPART_PART_SIZE = 5 * 1024 * 1024 +DEFAULT_MULTIPART_PART_SIZE = 128 * 1024 * 1024 +DEFAULT_COPY_PART_SIZE = 1024 * 1024 * 1024 +MAX_MULTIPART_PARTS = 10_000 +MAX_COPY_PART_SIZE = 5 * 1024 * 1024 * 1024 +OFFICIAL_NUPLAN_OPEN_DATA_BUCKET = "motional-nuplan" +OFFICIAL_NUPLAN_OPEN_DATA_PREFIX = "public/nuplan-v1.1/" +OFFICIAL_NUPLAN_OPEN_DATA_REGION = "ap-northeast-1" + +_ID_RE = re.compile(r"^[a-z0-9][a-z0-9._-]{0,127}$") +_FILENAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,255}$") +_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +_MD5_RE = re.compile(r"^[0-9a-f]{32}$") +_ETAG_RE = re.compile(r"^[0-9a-f]{32}(?:-[1-9][0-9]{0,4})?$") + + +def canonical_json_bytes(value: Any) -> bytes: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + +def sha256_bytes(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def _require_text(value: Any, field: str, *, pattern: re.Pattern[str] | None = None) -> str: + if not isinstance(value, str) or not value: + raise ValueError(f"{field} must be a non-empty string") + if pattern is not None and pattern.fullmatch(value) is None: + raise ValueError(f"{field} has an invalid value: {value!r}") + return value + + +def _validate_extract_to(value: Any, component: str) -> str: + path = _require_text(value, "archives[].extract_to") + parts = path.split("/") + if ( + path.startswith("/") + or path.endswith("/") + or any(part in {"", ".", ".."} for part in parts) + ): + raise ValueError( + f"archives[].extract_to must be a normalized relative path: {path!r}" + ) + required_prefix = { + "maps": "maps", + "database": "nuplan-v1.1/splits", + "sensor_blobs": "nuplan-v1.1/sensor_blobs", + }[component] + if path != required_prefix and not path.startswith(f"{required_prefix}/"): + raise ValueError( + f"{component} archive extract_to must be under {required_prefix!r}" + ) + return path + + +def _validate_source_uri(value: Any) -> str: + uri = _require_text(value, "archives[].source_uri") + parsed = urlsplit(uri) + if parsed.scheme == "s3": + if ( + not parsed.netloc + or not parsed.path.lstrip("/") + or parsed.query + or parsed.fragment + ): + raise ValueError("s3 source_uri must contain only bucket and object key") + return uri + if parsed.scheme != "https": + raise ValueError("source_uri must use https:// or s3://") + if ( + not parsed.hostname + or parsed.username is not None + or parsed.password is not None + or parsed.fragment + ): + raise ValueError( + "https source_uri must not contain credentials or a fragment" + ) + try: + address = ipaddress.ip_address(parsed.hostname) + except ValueError: + address = None + if address is not None and ( + address.is_private + or address.is_loopback + or address.is_link_local + or address.is_reserved + ): + raise ValueError("https source_uri must not target a private address") + return uri + + +def validate_resolved_https_host(hostname: str) -> None: + """Reject DNS rebinding to private networks before opening an HTTPS source.""" + for result in socket.getaddrinfo(hostname, 443, type=socket.SOCK_STREAM): + address = ipaddress.ip_address(result[4][0]) + if ( + address.is_private + or address.is_loopback + or address.is_link_local + or address.is_reserved + ): + raise ValueError( + f"HTTPS source hostname resolves to a private address: {hostname}" + ) + + +def validate_public_https_uri(uri: str) -> str: + """Validate an HTTPS source or redirect before opening a connection.""" + validated = _validate_source_uri(uri) + parsed = urlsplit(validated) + if parsed.scheme != "https": + raise ValueError("redirect target must use https://") + validate_resolved_https_host(parsed.hostname or "") + return validated + + +def normalize_s3_etag(value: Any, field: str = "ETag") -> str: + """Return a lowercase unquoted S3 ETag suitable for If-Match checks.""" + etag = _require_text(value, field).strip('"').lower() + if _ETAG_RE.fullmatch(etag) is None: + raise ValueError(f"{field} has an invalid S3 ETag: {value!r}") + return etag + + +def official_nuplan_open_data_region(bucket: str, key: str) -> str | None: + """Return the source region only for the pinned official nuPlan prefix.""" + if ( + bucket == OFFICIAL_NUPLAN_OPEN_DATA_BUCKET + and key.startswith(OFFICIAL_NUPLAN_OPEN_DATA_PREFIX) + ): + return OFFICIAL_NUPLAN_OPEN_DATA_REGION + return None + + +def validate_s3_source_head( + head: Mapping[str, Any], + archive: Mapping[str, Any], +) -> str: + """Validate one declared S3 source and return its quoted If-Match value.""" + content_length = head.get("ContentLength") + if ( + isinstance(content_length, bool) + or not isinstance(content_length, int) + or content_length != int(archive["expected_size_bytes"]) + ): + raise ValueError( + "source S3 object size differs for " + f"{archive['archive_id']!r}: expected " + f"{archive['expected_size_bytes']}, got {content_length}" + ) + actual_etag = normalize_s3_etag( + head.get("ETag"), + f"{archive['archive_id']}.source_etag", + ) + expected_etag = archive.get("expected_etag", "") + if expected_etag and actual_etag != expected_etag: + raise ValueError( + "source S3 object ETag differs for " + f"{archive['archive_id']!r}: expected {expected_etag}, " + f"got {actual_etag}" + ) + return f'"{actual_etag}"' + + +def validate_source_manifest(payload: Mapping[str, Any]) -> dict[str, Any]: + if payload.get("schema_version") != SOURCE_SCHEMA_VERSION: + raise ValueError( + f"source manifest schema_version must be {SOURCE_SCHEMA_VERSION!r}" + ) + snapshot_id = _require_text( + payload.get("snapshot_id"), + "snapshot_id", + pattern=_ID_RE, + ) + dataset_revision = _require_text( + payload.get("dataset_revision"), + "dataset_revision", + pattern=_ID_RE, + ) + map_version = _require_text( + payload.get("map_version"), + "map_version", + pattern=_ID_RE, + ) + if payload.get("terms_of_use_accepted") is not True: + raise ValueError("terms_of_use_accepted must be explicitly true") + authorization_reference = _require_text( + payload.get("authorization_reference"), + "authorization_reference", + ) + if len(authorization_reference) > 256: + raise ValueError("authorization_reference must be at most 256 characters") + + raw_archives = payload.get("archives") + if not isinstance(raw_archives, list) or not raw_archives: + raise ValueError("archives must be a non-empty list") + archives: list[dict[str, Any]] = [] + archive_ids: set[str] = set() + filenames: set[tuple[str, str]] = set() + for index, raw_archive in enumerate(raw_archives): + if not isinstance(raw_archive, Mapping): + raise ValueError(f"archives[{index}] must be an object") + archive_id = _require_text( + raw_archive.get("archive_id"), + f"archives[{index}].archive_id", + pattern=_ID_RE, + ) + if archive_id in archive_ids: + raise ValueError(f"duplicate archive_id {archive_id!r}") + archive_ids.add(archive_id) + component = _require_text( + raw_archive.get("component"), + f"archives[{index}].component", + ) + if component not in REQUIRED_COMPONENTS: + raise ValueError( + f"archives[{index}].component must be one of " + f"{sorted(REQUIRED_COMPONENTS)}" + ) + filename = _require_text( + raw_archive.get("filename"), + f"archives[{index}].filename", + pattern=_FILENAME_RE, + ) + filename_key = (component, filename) + if filename_key in filenames: + raise ValueError( + f"duplicate filename {filename!r} for component {component!r}" + ) + filenames.add(filename_key) + expected_size_bytes = raw_archive.get("expected_size_bytes") + if ( + isinstance(expected_size_bytes, bool) + or not isinstance(expected_size_bytes, int) + or expected_size_bytes <= 0 + ): + raise ValueError( + f"archives[{index}].expected_size_bytes must be positive" + ) + expected_sha256 = raw_archive.get("expected_sha256", "") + expected_md5 = raw_archive.get("expected_md5", "") + expected_etag = raw_archive.get("expected_etag", "") + if expected_sha256: + _require_text( + expected_sha256, + f"archives[{index}].expected_sha256", + pattern=_SHA256_RE, + ) + if expected_md5: + _require_text( + expected_md5, + f"archives[{index}].expected_md5", + pattern=_MD5_RE, + ) + if expected_etag: + expected_etag = normalize_s3_etag( + expected_etag, + f"archives[{index}].expected_etag", + ) + source_uri = _validate_source_uri(raw_archive.get("source_uri")) + source_scheme = urlsplit(source_uri).scheme + if expected_etag and source_scheme != "s3": + raise ValueError( + f"archives[{index}].expected_etag is only valid for s3:// sources" + ) + if source_scheme == "s3" and not expected_etag: + raise ValueError( + f"archives[{index}] s3 source must declare expected_etag" + ) + if source_scheme == "s3" and (expected_sha256 or expected_md5): + raise ValueError( + f"archives[{index}] s3 source must use expected_etag " + "without expected_sha256 or expected_md5" + ) + if source_scheme == "https" and not expected_sha256 and not expected_md5: + raise ValueError( + f"archives[{index}] https source must declare " + "expected_sha256 or expected_md5" + ) + archives.append({ + "archive_id": archive_id, + "component": component, + "expected_etag": expected_etag, + "expected_md5": expected_md5, + "expected_sha256": expected_sha256, + "expected_size_bytes": expected_size_bytes, + "extract_to": _validate_extract_to( + raw_archive.get("extract_to"), + component, + ), + "filename": filename, + "source_uri": source_uri, + }) + + components = {archive["component"] for archive in archives} + missing_components = REQUIRED_COMPONENTS - components + if missing_components: + raise ValueError( + "source manifest is missing required components: " + f"{sorted(missing_components)}" + ) + return { + "archives": archives, + "authorization_reference": authorization_reference, + "dataset_revision": dataset_revision, + "map_version": map_version, + "schema_version": SOURCE_SCHEMA_VERSION, + "snapshot_id": snapshot_id, + "terms_of_use_accepted": True, + } + + +def load_source_manifest_bytes(payload: bytes) -> tuple[dict[str, Any], str]: + try: + parsed = json.loads(payload) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise ValueError("nuPlan source manifest is not valid UTF-8 JSON") from error + if not isinstance(parsed, Mapping): + raise ValueError("nuPlan source manifest root must be an object") + manifest = validate_source_manifest(parsed) + contract = { + **manifest, + "archives": [ + { + key: value + for key, value in archive.items() + if key != "source_uri" + } + for archive in manifest["archives"] + ], + } + return manifest, sha256_bytes(canonical_json_bytes(contract)) + + +def snapshot_prefix(source_manifest: Mapping[str, Any]) -> str: + return ( + "nuplan/raw-snapshots/" + f"{source_manifest['dataset_revision']}/{source_manifest['snapshot_id']}" + ) + + +def archive_object_key( + source_manifest: Mapping[str, Any], + archive: Mapping[str, Any], +) -> str: + return ( + f"{snapshot_prefix(source_manifest)}/archives/" + f"{archive['component']}/{archive['archive_id']}/{archive['filename']}" + ) + + +def archive_receipt_key( + source_manifest: Mapping[str, Any], + archive: Mapping[str, Any], +) -> str: + return f"{archive_object_key(source_manifest, archive)}.receipt.json" + + +def snapshot_manifest_key(source_manifest: Mapping[str, Any]) -> str: + return f"{snapshot_prefix(source_manifest)}/manifest.json" + + +def digest_stream( + stream: BinaryIO, + *, + chunk_size: int = 8 * 1024 * 1024, +) -> dict[str, Any]: + if chunk_size <= 0: + raise ValueError("chunk_size must be positive") + sha256 = hashlib.sha256() + md5 = hashlib.md5(usedforsecurity=False) + total_size = 0 + while True: + chunk = stream.read(chunk_size) + if not chunk: + break + sha256.update(chunk) + md5.update(chunk) + total_size += len(chunk) + return { + "md5": md5.hexdigest(), + "sha256": sha256.hexdigest(), + "size_bytes": total_size, + } + + +def validate_archive_digest( + result: Mapping[str, Any], + *, + expected_size_bytes: int, + expected_sha256: str = "", + expected_md5: str = "", + label: str, +) -> None: + if int(result["size_bytes"]) != expected_size_bytes: + raise ValueError( + f"archive size mismatch for {label}: " + f"expected {expected_size_bytes}, got {result['size_bytes']}" + ) + if expected_sha256 and result["sha256"] != expected_sha256: + raise ValueError( + f"archive SHA-256 mismatch for {label}: " + f"expected {expected_sha256}, got {result['sha256']}" + ) + if expected_md5 and result["md5"] != expected_md5: + raise ValueError( + f"archive MD5 mismatch for {label}: " + f"expected {expected_md5}, got {result['md5']}" + ) + + +def upload_https_stream_multipart( + *, + s3_client: Any, + stream: BinaryIO, + bucket: str, + key: str, + metadata: Mapping[str, str], + expected_size_bytes: int, + expected_sha256: str = "", + expected_md5: str = "", + part_size: int = DEFAULT_MULTIPART_PART_SIZE, +) -> dict[str, Any]: + """Stream one HTTPS archive to S3 while verifying its declared integrity.""" + if part_size < MIN_MULTIPART_PART_SIZE: + raise ValueError("part_size must satisfy the S3 multipart minimum") + required_parts = (expected_size_bytes + part_size - 1) // part_size + if required_parts > MAX_MULTIPART_PARTS: + raise ValueError( + "archive requires more than the S3 multipart limit of " + f"{MAX_MULTIPART_PARTS} parts" + ) + create_response = s3_client.create_multipart_upload( + Bucket=bucket, + Key=key, + ContentType="application/octet-stream", + Metadata=dict(metadata), + ) + upload_id = create_response["UploadId"] + sha256 = hashlib.sha256() + md5 = hashlib.md5(usedforsecurity=False) + total_size = 0 + parts: list[dict[str, Any]] = [] + try: + part_number = 1 + while True: + buffer = bytearray() + while len(buffer) < part_size: + chunk = stream.read(part_size - len(buffer)) + if not chunk: + break + buffer.extend(chunk) + if not buffer: + break + if part_number > MAX_MULTIPART_PARTS: + raise ValueError( + "archive exceeded the S3 multipart limit of " + f"{MAX_MULTIPART_PARTS} parts" + ) + payload = bytes(buffer) + sha256.update(payload) + md5.update(payload) + total_size += len(payload) + response = s3_client.upload_part( + Bucket=bucket, + Key=key, + UploadId=upload_id, + PartNumber=part_number, + Body=payload, + ) + parts.append({ + "ETag": response["ETag"], + "PartNumber": part_number, + }) + part_number += 1 + + actual_sha256 = sha256.hexdigest() + actual_md5 = md5.hexdigest() + result = { + "md5": actual_md5, + "sha256": actual_sha256, + "size_bytes": total_size, + } + validate_archive_digest( + result, + expected_size_bytes=expected_size_bytes, + expected_sha256=expected_sha256, + expected_md5=expected_md5, + label=key, + ) + if not parts: + raise ValueError(f"archive source is empty for {key}") + s3_client.complete_multipart_upload( + Bucket=bucket, + Key=key, + UploadId=upload_id, + MultipartUpload={"Parts": parts}, + ) + except BaseException: + s3_client.abort_multipart_upload( + Bucket=bucket, + Key=key, + UploadId=upload_id, + ) + raise + return result + + +def copy_s3_object_multipart( + *, + s3_client: Any, + source_bucket: str, + source_key: str, + source_etag: str, + destination_bucket: str, + destination_key: str, + metadata: Mapping[str, str], + expected_size_bytes: int, + part_size: int = DEFAULT_COPY_PART_SIZE, +) -> dict[str, Any]: + """Copy one pinned S3 object without routing data through the caller.""" + if part_size < MIN_MULTIPART_PART_SIZE: + raise ValueError("part_size must satisfy the S3 multipart minimum") + if part_size > MAX_COPY_PART_SIZE: + raise ValueError("part_size must not exceed the S3 copy-part maximum") + if expected_size_bytes <= 0: + raise ValueError("expected_size_bytes must be positive") + required_parts = (expected_size_bytes + part_size - 1) // part_size + if required_parts > MAX_MULTIPART_PARTS: + raise ValueError( + "archive requires more than the S3 multipart limit of " + f"{MAX_MULTIPART_PARTS} parts" + ) + normalized_source_etag = normalize_s3_etag( + source_etag, + "source_etag", + ) + create_response = s3_client.create_multipart_upload( + Bucket=destination_bucket, + Key=destination_key, + ContentType="application/octet-stream", + Metadata=dict(metadata), + ChecksumAlgorithm="CRC64NVME", + ChecksumType="FULL_OBJECT", + ) + upload_id = create_response["UploadId"] + parts: list[dict[str, Any]] = [] + try: + for part_number in range(1, required_parts + 1): + start = (part_number - 1) * part_size + end = min(expected_size_bytes - 1, start + part_size - 1) + response = s3_client.upload_part_copy( + Bucket=destination_bucket, + Key=destination_key, + UploadId=upload_id, + PartNumber=part_number, + CopySource={ + "Bucket": source_bucket, + "Key": source_key, + }, + CopySourceIfMatch=f'"{normalized_source_etag}"', + CopySourceRange=f"bytes={start}-{end}", + ) + parts.append({ + "ETag": response["CopyPartResult"]["ETag"], + "PartNumber": part_number, + }) + completed = s3_client.complete_multipart_upload( + Bucket=destination_bucket, + Key=destination_key, + UploadId=upload_id, + MultipartUpload={"Parts": parts}, + MpuObjectSize=expected_size_bytes, + ChecksumType="FULL_OBJECT", + ) + except BaseException: + s3_client.abort_multipart_upload( + Bucket=destination_bucket, + Key=destination_key, + UploadId=upload_id, + ) + raise + + head = s3_client.head_object( + Bucket=destination_bucket, + Key=destination_key, + ChecksumMode="ENABLED", + ) + if int(head["ContentLength"]) != expected_size_bytes: + raise ValueError( + "server-side copied object size differs for " + f"{destination_key}: expected {expected_size_bytes}, " + f"got {head['ContentLength']}" + ) + checksum = head.get("ChecksumCRC64NVME") + if not isinstance(checksum, str) or not checksum: + raise ValueError( + f"server-side copied object lacks CRC64NVME: {destination_key}" + ) + completed_checksum = completed.get("ChecksumCRC64NVME") + if completed_checksum and completed_checksum != checksum: + raise ValueError( + "server-side copied object checksum differs after completion: " + f"{destination_key}" + ) + return { + "checksum_crc64nvme": checksum, + "destination_etag": str(head["ETag"]).strip('"').lower(), + "md5": "", + "sha256": "", + "size_bytes": expected_size_bytes, + "source_etag": normalized_source_etag, + "transfer_mode": "s3_server_side_multipart_copy", + } + + +def build_snapshot_manifest( + *, + source_manifest: Mapping[str, Any], + source_contract_sha256: str, + receipts: Sequence[Mapping[str, Any]], +) -> dict[str, Any]: + if not _SHA256_RE.fullmatch(source_contract_sha256): + raise ValueError("source_contract_sha256 must be lowercase SHA-256") + by_archive_id: dict[str, Mapping[str, Any]] = {} + for receipt in receipts: + if receipt.get("schema_version") != ARCHIVE_RECEIPT_SCHEMA_VERSION: + raise ValueError("archive receipt has an unsupported schema") + archive_id = _require_text( + receipt.get("archive_id"), + "receipt.archive_id", + pattern=_ID_RE, + ) + if archive_id in by_archive_id: + raise ValueError(f"duplicate receipt for archive {archive_id!r}") + if receipt.get("source_contract_sha256") != source_contract_sha256: + raise ValueError( + f"archive receipt {archive_id!r} has the wrong source contract" + ) + if "source_uri" in receipt: + raise ValueError("archive receipt must not disclose source_uri") + by_archive_id[archive_id] = receipt + + expected_ids = { + archive["archive_id"] for archive in source_manifest["archives"] + } + if set(by_archive_id) != expected_ids: + raise ValueError( + "archive receipts do not exactly match the source manifest" + ) + archives = [] + for archive in source_manifest["archives"]: + receipt = by_archive_id[archive["archive_id"]] + if int(receipt["size_bytes"]) != int(archive["expected_size_bytes"]): + raise ValueError( + f"receipt size mismatch for {archive['archive_id']!r}" + ) + if ( + archive["expected_sha256"] + and receipt["sha256"] != archive["expected_sha256"] + ): + raise ValueError( + f"receipt SHA-256 mismatch for {archive['archive_id']!r}" + ) + if archive["expected_md5"] and receipt["md5"] != archive["expected_md5"]: + raise ValueError(f"receipt MD5 mismatch for {archive['archive_id']!r}") + if receipt.get("transfer_mode") == "s3_server_side_multipart_copy": + if receipt.get("source_etag") != archive["expected_etag"]: + raise ValueError( + f"receipt source ETag mismatch for {archive['archive_id']!r}" + ) + if not receipt.get("checksum_crc64nvme"): + raise ValueError( + "server-side copy receipt lacks CRC64NVME for " + f"{archive['archive_id']!r}" + ) + archives.append({ + "archive_id": archive["archive_id"], + "checksum_crc64nvme": receipt.get("checksum_crc64nvme", ""), + "component": archive["component"], + "destination_etag": receipt.get("destination_etag", ""), + "extract_to": archive["extract_to"], + "filename": archive["filename"], + "md5": receipt["md5"], + "object_uri": receipt["object_uri"], + "sha256": receipt["sha256"], + "size_bytes": int(receipt["size_bytes"]), + "transfer_mode": receipt.get( + "transfer_mode", + "legacy_stream_hash", + ), + **( + {"source_etag": archive["expected_etag"]} + if archive["expected_etag"] + else {} + ), + }) + + archives.sort(key=lambda item: item["archive_id"]) + component_counts = { + component: sum( + archive["component"] == component for archive in archives + ) + for component in sorted(REQUIRED_COMPONENTS) + } + return { + "archives": archives, + "authorization_reference": source_manifest["authorization_reference"], + "component_counts": component_counts, + "dataset": "nuplan/nuplan-v1.1", + "dataset_revision": source_manifest["dataset_revision"], + "map_version": source_manifest["map_version"], + "schema_version": SNAPSHOT_SCHEMA_VERSION, + "snapshot_id": source_manifest["snapshot_id"], + "source_contract_sha256": source_contract_sha256, + "terms_of_use_accepted": True, + "total_size_bytes": sum( + int(archive["size_bytes"]) for archive in archives + ), + } diff --git a/Platform/pipelines/occupancy_store.py b/Platform/pipelines/occupancy_store.py new file mode 100644 index 000000000..9099b16e4 --- /dev/null +++ b/Platform/pipelines/occupancy_store.py @@ -0,0 +1,339 @@ +"""Immutable publication contract for semantic occupancy model sets.""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections.abc import Mapping, Sequence +from typing import Any + +OCCUPANCY_SET_SCHEMA = "semantic_occupancy_set_v2" +OCCUPANCY_SET_PREFIX = "semantic-occupancy-sets/schema=v2" +OCCUPANCY_ARTIFACT_KINDS = frozenset({ + "native-semantic-occupancy", + "detection-derived-occupancy", +}) +_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +_VERSION_RE = re.compile(r"^v[1-9][0-9]*\.[0-9]+$") +_SEGMENT_RE = re.compile(r"^[A-Za-z0-9_.-]+$") + + +def _sha256(value: str, label: str) -> str: + if not _SHA256_RE.fullmatch(value): + raise ValueError(f"{label} must be a lowercase SHA-256") + return value + + +def _segment(value: str, label: str) -> str: + if not value or not _SEGMENT_RE.fullmatch(value): + raise ValueError(f"{label} must be one canonical path segment") + return value + + +def occupancy_set_s3_key( + dataset: str, + dataset_version: str, + model_artifact_id: str, + dataset_manifest_sha256: str, +) -> str: + """Return the dataset-first discovery key written last by a producer.""" + dataset = _segment(dataset, "dataset") + if not _VERSION_RE.fullmatch(dataset_version): + raise ValueError("dataset_version must match v.") + model_artifact_id = _sha256(model_artifact_id, "model_artifact_id") + dataset_manifest_sha256 = _sha256( + dataset_manifest_sha256, + "dataset_manifest_sha256", + ) + return ( + f"{OCCUPANCY_SET_PREFIX}/dataset={dataset}/" + f"version={dataset_version}/model={model_artifact_id}/" + f"manifest={dataset_manifest_sha256}/manifest.json" + ) + + +def _string_list( + values: Sequence[str], + label: str, + *, + allow_empty: bool = False, +) -> list[str]: + output = [str(value) for value in values] + if (not allow_empty and not output) or any( + not value or value.strip() != value for value in output + ): + raise ValueError(f"{label} must contain non-empty trimmed strings") + if len(set(output)) != len(output): + raise ValueError(f"{label} must not contain duplicates") + return output + + +def _model_source(model_source: Mapping[str, Any]) -> dict[str, str]: + source = { + str(key): value + for key, value in model_source.items() + } + required_source = { + "code_license_spdx", + "config", + "license_spdx", + "repository", + "repository_revision", + "training_data_license_spdx", + "weight_sha256", + "weight_source_url", + } + if set(source) != required_source: + raise ValueError( + "model_source must contain exactly " + + ", ".join(sorted(required_source)) + ) + if any( + not isinstance(source[key], str) + or not source[key] + or source[key].strip() != source[key] + for key in required_source + ): + raise ValueError("model_source values must be non-empty strings") + _sha256(source["weight_sha256"], "model_source.weight_sha256") + return source + + +def _producer_config(producer_config: Mapping[str, Any]) -> dict[str, Any]: + if not producer_config or any( + not isinstance(key, str) + or not key + or key.strip() != key + for key in producer_config + ): + raise ValueError( + "producer_config must have non-empty trimmed string keys" + ) + try: + payload = json.dumps( + producer_config, + allow_nan=False, + separators=(",", ":"), + sort_keys=True, + ) + except (TypeError, ValueError) as error: + raise ValueError("producer_config must be canonical JSON") from error + normalized = json.loads(payload) + if not isinstance(normalized, dict): + raise ValueError("producer_config must be a JSON object") + return normalized + + +def occupancy_model_artifact_id( + *, + artifact_kind: str, + artifact_schema: str, + geometry_id: str, + head_version: str, + input_contract: str, + model_source: Mapping[str, Any], + producer_config: Mapping[str, Any], + taxonomy_version: str, +) -> str: + """Hash every dataset-independent input that determines an ASOC body.""" + if artifact_kind not in OCCUPANCY_ARTIFACT_KINDS: + raise ValueError(f"unsupported artifact_kind {artifact_kind!r}") + for label, value in ( + ("artifact_schema", artifact_schema), + ("geometry_id", geometry_id), + ("head_version", head_version), + ("input_contract", input_contract), + ("taxonomy_version", taxonomy_version), + ): + if not value or value.strip() != value: + raise ValueError(f"{label} must be a non-empty trimmed string") + identity = { + "artifact_kind": artifact_kind, + "artifact_schema": artifact_schema, + "geometry_id": geometry_id, + "head_version": head_version, + "input_contract": input_contract, + "model_source": _model_source(model_source), + "producer_config": _producer_config(producer_config), + "taxonomy_version": taxonomy_version, + } + payload = json.dumps( + identity, + allow_nan=False, + separators=(",", ":"), + sort_keys=True, + ).encode("ascii") + return hashlib.sha256(payload).hexdigest() + + +def occupancy_set_manifest( + *, + artifact_kind: str, + artifact_schema: str, + created_at: str, + dataset: str, + dataset_version: str, + dataset_manifest_sha256: str, + display_name: str, + geometry_id: str, + head_version: str, + input_contract: str, + limitations: Sequence[str], + model_artifact_id: str, + model_family: str, + model_source: Mapping[str, Any], + producer_config: Mapping[str, Any], + shards: Sequence[Mapping[str, Any]], + supported_classes: Sequence[str], + taxonomy_version: str, + teacher_available: bool, +) -> dict[str, Any]: + """Build and validate the manifest that atomically publishes one model.""" + if artifact_kind not in OCCUPANCY_ARTIFACT_KINDS: + raise ValueError(f"unsupported artifact_kind {artifact_kind!r}") + model_artifact_id = _sha256(model_artifact_id, "model_artifact_id") + dataset_manifest_sha256 = _sha256( + dataset_manifest_sha256, + "dataset_manifest_sha256", + ) + occupancy_set_s3_key( + dataset, + dataset_version, + model_artifact_id, + dataset_manifest_sha256, + ) + for label, value in ( + ("artifact_schema", artifact_schema), + ("created_at", created_at), + ("display_name", display_name), + ("geometry_id", geometry_id), + ("head_version", head_version), + ("input_contract", input_contract), + ("model_family", model_family), + ("taxonomy_version", taxonomy_version), + ): + if not value or value.strip() != value: + raise ValueError(f"{label} must be a non-empty trimmed string") + + source = _model_source(model_source) + normalized_producer_config = _producer_config(producer_config) + expected_artifact_id = occupancy_model_artifact_id( + artifact_kind=artifact_kind, + artifact_schema=artifact_schema, + geometry_id=geometry_id, + head_version=head_version, + input_contract=input_contract, + model_source=source, + producer_config=normalized_producer_config, + taxonomy_version=taxonomy_version, + ) + if model_artifact_id != expected_artifact_id: + raise ValueError( + "model_artifact_id does not identify the complete producer recipe" + ) + + supported = _string_list( + supported_classes, + "supported_classes", + ) + disclosed_limitations = _string_list( + limitations, + "limitations", + ) + entries = [] + seen_shards: set[str] = set() + total_samples = 0 + for raw_entry in shards: + required_entry = { + "byte_size", + "s3_key", + "sample_count", + "sha256", + "shard", + "teacher_present", + } + if set(raw_entry) != required_entry: + raise ValueError( + "occupancy shard entry has an unexpected field set" + ) + shard = _segment(str(raw_entry["shard"]), "shard") + if shard in seen_shards: + raise ValueError(f"duplicate occupancy shard {shard!r}") + seen_shards.add(shard) + payload_sha256 = _sha256(str(raw_entry["sha256"]), "shard sha256") + byte_size = int(raw_entry["byte_size"]) + sample_count = int(raw_entry["sample_count"]) + teacher_present = bool(raw_entry["teacher_present"]) + if byte_size <= 0 or sample_count <= 0: + raise ValueError("occupancy shard sizes must be positive") + if teacher_present != teacher_available: + raise ValueError( + "shard teacher availability differs from the model set" + ) + key = str(raw_entry["s3_key"]) + canonical_prefix = ( + f"semantic-occupancy/schema={artifact_schema}/" + f"model={model_artifact_id}/" + f"manifest={dataset_manifest_sha256}/" + f"geometry={geometry_id}/taxonomy={taxonomy_version}/" + f"head={head_version}/dataset={dataset}/shard={shard}/" + ) + if ( + not key.startswith(canonical_prefix) + or not key.endswith("/occupancy.bin.gz") + ): + raise ValueError("occupancy shard key is not canonical") + entries.append({ + "byte_size": byte_size, + "s3_key": key, + "sample_count": sample_count, + "sha256": payload_sha256, + "shard": shard, + "teacher_present": teacher_present, + }) + total_samples += sample_count + if not entries: + raise ValueError("occupancy set must contain at least one shard") + entries.sort(key=lambda entry: entry["shard"]) + + return { + "schema_version": OCCUPANCY_SET_SCHEMA, + "artifact_kind": artifact_kind, + "artifact_schema": artifact_schema, + "created_at": created_at, + "dataset": dataset, + "dataset_version": dataset_version, + "dataset_manifest_sha256": dataset_manifest_sha256, + "display_name": display_name, + "geometry_id": geometry_id, + "head_version": head_version, + "input_contract": input_contract, + "limitations": disclosed_limitations, + "model_artifact_id": model_artifact_id, + "model_family": model_family, + "model_source": source, + "producer_config": normalized_producer_config, + "sample_count": total_samples, + "shard_count": len(entries), + "shards": entries, + "supported_classes": supported, + "taxonomy_version": taxonomy_version, + "teacher_available": teacher_available, + } + + +def encode_occupancy_set_manifest( + manifest: Mapping[str, Any], +) -> tuple[bytes, str]: + """Return stable ASCII JSON and its content digest.""" + payload = ( + json.dumps( + manifest, + allow_nan=False, + indent=2, + sort_keys=True, + ) + + "\n" + ).encode("ascii") + return payload, hashlib.sha256(payload).hexdigest() diff --git a/Platform/pipelines/semantic_occupancy.py b/Platform/pipelines/semantic_occupancy.py new file mode 100644 index 000000000..9336086db --- /dev/null +++ b/Platform/pipelines/semantic_occupancy.py @@ -0,0 +1,412 @@ +"""Immutable semantic occupancy artifact and offline inference.""" + +from __future__ import annotations + +import gzip +import hashlib +import io +import re +import struct +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np +import torch + +from navigation.geometry import AUTOE2E_NAVIGATION_GEOMETRY + +SEMANTIC_OCCUPANCY_SCHEMA = "v1" +SEMANTIC_OCCUPANCY_FORMAT_VERSION = 1 +SEMANTIC_OCCUPANCY_MAGIC = b"ASOC" +SEMANTIC_OCCUPANCY_TAXONOMY_VERSION = "autoe2e-bev-semantic-v1" +SEMANTIC_OCCUPANCY_GEOMETRY_ID = ( + AUTOE2E_NAVIGATION_GEOMETRY.geometry_id +) +SEMANTIC_OCCUPANCY_HEAD_VERSION = "bev-segmentation-head-v1" +SEMANTIC_OCCUPANCY_CLASS_NAMES = ( + "drivable_area", + "lane_area", + "intersection", + "crosswalk", + "stop_line", + "vehicle", + "vulnerable_road_user", + "other_obstacle", +) +FLAG_TEACHER_PRESENT = 1 << 0 +_HEADER = struct.Struct("<4sHHIHHHH") +_DIRECTORY_ENTRY = struct.Struct(" int: + digest = hashlib.sha256(sample_uid.encode("utf-8")).digest() + return int.from_bytes(digest[:8], "little", signed=False) + + +def semantic_occupancy_s3_key( + model_checkpoint_sha256: str, + dataset_manifest_sha256: str, + dataset: str, + shard: str, + *, + artifact_schema: str = SEMANTIC_OCCUPANCY_SCHEMA, + geometry_id: str = SEMANTIC_OCCUPANCY_GEOMETRY_ID, + taxonomy_version: str = SEMANTIC_OCCUPANCY_TAXONOMY_VERSION, + head_version: str = SEMANTIC_OCCUPANCY_HEAD_VERSION, +) -> str: + for label, digest in ( + ("model_checkpoint_sha256", model_checkpoint_sha256), + ("dataset_manifest_sha256", dataset_manifest_sha256), + ): + if len(digest) != 64 or any( + character not in "0123456789abcdef" + for character in digest + ): + raise ValueError(f"{label} must be lowercase SHA-256") + for label, value in ( + ("artifact_schema", artifact_schema), + ("geometry_id", geometry_id), + ("taxonomy_version", taxonomy_version), + ("head_version", head_version), + ("dataset", dataset), + ("shard", shard), + ): + if not _PATH_SEGMENT_RE.fullmatch(value): + raise ValueError(f"{label} must be one canonical path segment") + return ( + f"semantic-occupancy/schema={artifact_schema}/" + f"model={model_checkpoint_sha256}/" + f"manifest={dataset_manifest_sha256}/" + f"geometry={geometry_id}/" + f"taxonomy={taxonomy_version}/" + f"head={head_version}/dataset={dataset}/" + f"shard={shard}/occupancy.bin.gz" + ) + + +def quantize_semantic_occupancy( + probability: np.ndarray, +) -> np.ndarray: + """Return canonical uint8 cells for one frame or a frame batch.""" + values = np.asarray(probability) + if values.ndim == 3: + class_count = values.shape[0] + expected_shape = "[8,H,W]" + elif values.ndim == 4: + class_count = values.shape[1] + expected_shape = "[N,8,H,W]" + else: + class_count = -1 + expected_shape = "[8,H,W] or [N,8,H,W]" + if class_count != len(SEMANTIC_OCCUPANCY_CLASS_NAMES): + raise ValueError(f"probability must have shape {expected_shape}") + if values.dtype == np.uint8: + return np.ascontiguousarray(values) + if ( + not np.issubdtype(values.dtype, np.floating) + or not np.isfinite(values).all() + or np.any(values < 0.0) + or np.any(values > 1.0) + ): + raise ValueError("probability must be finite floating point in [0,1]") + return np.ascontiguousarray( + np.rint(values * 255.0), + dtype=np.uint8, + ) + + +def encode_semantic_occupancy( + sample_uids: Sequence[str], + probability: np.ndarray, + *, + teacher: np.ndarray | None = None, + valid_mask: np.ndarray | None = None, +) -> bytes: + sample_uids = tuple(sample_uids) + if not sample_uids or len(set(sample_uids)) != len(sample_uids): + raise ValueError("sample UIDs must be non-empty and unique") + probability_u8 = quantize_semantic_occupancy(probability) + if probability_u8.ndim != 4: + raise ValueError("probability must have shape [N,8,H,W]") + sample_count, class_count, height, width = probability_u8.shape + if sample_count != len(sample_uids): + raise ValueError("sample UID count differs from probability rows") + if max(sample_count, class_count, height, width) > 0xFFFF: + raise ValueError("semantic occupancy dimensions exceed uint16") + + flags = 0 + teacher_u8 = None + valid_bits = None + if (teacher is None) != (valid_mask is None): + raise ValueError("teacher and valid_mask must be present together") + if teacher is not None and valid_mask is not None: + teacher_u8 = quantize_semantic_occupancy(teacher) + valid = np.asarray(valid_mask, dtype=np.bool_) + if teacher_u8.shape != probability_u8.shape or valid.shape != ( + probability_u8.shape + ): + raise ValueError("teacher tensors must match probability shape") + valid_bits = np.packbits( + valid.reshape(-1), + bitorder="little", + ) + flags |= FLAG_TEACHER_PRESENT + + hashes = [sample_uid_hash(uid) for uid in sample_uids] + if len(set(hashes)) != len(hashes): + raise ValueError("sample UID hash collision") + directory = sorted( + (uid_hash, row) + for row, uid_hash in enumerate(hashes) + ) + raw = io.BytesIO() + raw.write(_HEADER.pack( + SEMANTIC_OCCUPANCY_MAGIC, + SEMANTIC_OCCUPANCY_FORMAT_VERSION, + flags, + sample_count, + class_count, + height, + width, + 0, + )) + for uid_hash, row in directory: + raw.write(_DIRECTORY_ENTRY.pack(uid_hash, row)) + raw.write(probability_u8.tobytes(order="C")) + if teacher_u8 is not None and valid_bits is not None: + raw.write(teacher_u8.tobytes(order="C")) + raw.write(valid_bits.tobytes(order="C")) + + compressed = io.BytesIO() + with gzip.GzipFile( + filename="", + mode="wb", + fileobj=compressed, + compresslevel=6, + mtime=0, + ) as stream: + stream.write(raw.getvalue()) + return compressed.getvalue() + + +def decode_semantic_occupancy( + payload: bytes, +) -> DecodedSemanticOccupancy: + try: + raw = gzip.decompress(payload) + except (EOFError, OSError) as exc: + raise ValueError("semantic occupancy artifact is not valid gzip") from exc + if len(raw) < _HEADER.size: + raise ValueError("semantic occupancy artifact is truncated") + ( + magic, + version, + flags, + sample_count, + class_count, + height, + width, + reserved, + ) = _HEADER.unpack_from(raw) + if ( + magic != SEMANTIC_OCCUPANCY_MAGIC + or version != SEMANTIC_OCCUPANCY_FORMAT_VERSION + or class_count != len(SEMANTIC_OCCUPANCY_CLASS_NAMES) + or reserved != 0 + or sample_count == 0 + or height == 0 + or width == 0 + or flags & ~FLAG_TEACHER_PRESENT + ): + raise ValueError("unsupported semantic occupancy header") + cell_count = sample_count * class_count * height * width + valid_byte_count = (cell_count + 7) // 8 + expected_size = ( + _HEADER.size + + sample_count * _DIRECTORY_ENTRY.size + + cell_count + ) + if flags & FLAG_TEACHER_PRESENT: + expected_size += cell_count + valid_byte_count + if len(raw) != expected_size: + raise ValueError("semantic occupancy artifact size mismatch") + + cursor = _HEADER.size + directory = [] + for _ in range(sample_count): + directory.append(_DIRECTORY_ENTRY.unpack_from(raw, cursor)) + cursor += _DIRECTORY_ENTRY.size + if directory != sorted(directory) or sorted( + row for _, row in directory + ) != list(range(sample_count)): + raise ValueError("semantic occupancy directory is invalid") + + shape = (sample_count, class_count, height, width) + probability = ( + np.frombuffer(raw, dtype=np.uint8, count=cell_count, offset=cursor) + .reshape(shape) + .astype(np.float32) + / 255.0 + ) + cursor += cell_count + teacher = None + valid_mask = None + if flags & FLAG_TEACHER_PRESENT: + teacher = ( + np.frombuffer( + raw, + dtype=np.uint8, + count=cell_count, + offset=cursor, + ) + .reshape(shape) + .astype(np.float32) + / 255.0 + ) + cursor += cell_count + valid_mask = np.unpackbits( + np.frombuffer( + raw, + dtype=np.uint8, + count=valid_byte_count, + offset=cursor, + ), + count=cell_count, + bitorder="little", + ).astype(np.bool_).reshape(shape) + return DecodedSemanticOccupancy( + flags=flags, + height=height, + width=width, + directory=tuple(directory), + probability=probability, + teacher=teacher, + valid_mask=valid_mask, + ) + + +def infer_semantic_occupancy( + model: torch.nn.Module, + loader: Any, + *, + device: torch.device, +) -> tuple[list[str], np.ndarray, np.ndarray | None, np.ndarray | None]: + """Precompute dense probabilities and optional teacher tensors.""" + was_training = model.training + sample_uids: list[str] = [] + probabilities = [] + teachers = [] + valid_masks = [] + teacher_mode: bool | None = None + from training.reactive_stage_runner import ( + resolve_reactive_batch_projection, + ) + + model.eval() + try: + with torch.no_grad(): + for item in loader: + if isinstance(item, tuple): + batch, projection, geometry_type = item + else: + batch, projection, geometry_type = item, None, "pseudo" + projection, geometry_type = ( + resolve_reactive_batch_projection( + batch, + projection, + geometry_type, + device=device, + ) + ) + output = model( + batch["visual_tiles"].to(device), + batch["map_context"].to(device), + batch["visual_history"].to(device), + batch["egomotion_history"].to(device), + route_mask=batch["route_mask"].to(device), + map_valid=batch["map_valid"].to(device), + route_valid=batch["route_valid"].to(device), + projection=projection, + geometry_type=geometry_type, + mode="infer", + return_auxiliary=True, + compute_bev_segmentation=True, + compute_route_reconstruction=False, + ) + if not isinstance(output, tuple): + raise TypeError( + "model did not emit semantic occupancy logits" + ) + _, auxiliary = output + logits = auxiliary.get("bev_segmentation_logits") + if not torch.is_tensor(logits): + raise RuntimeError( + "checkpoint has no BEV segmentation head" + ) + probabilities.append( + logits.sigmoid().float().cpu().numpy() + ) + batch_uids = [str(uid) for uid in batch["sample_uid"]] + sample_uids.extend(batch_uids) + available = batch.get("bev_segmentation_available") + has_teacher = ( + available is not None + and bool(torch.as_tensor(available).all()) + ) + if teacher_mode is None: + teacher_mode = has_teacher + elif teacher_mode != has_teacher: + raise ValueError( + "semantic artifact cannot mix teacher availability" + ) + if has_teacher: + teachers.append( + batch["bev_segmentation_target"].numpy() + ) + valid_masks.append( + batch["bev_segmentation_valid"].numpy() + ) + finally: + model.train(was_training) + if not sample_uids: + raise ValueError("semantic occupancy loader yielded no samples") + return ( + sample_uids, + np.concatenate(probabilities, axis=0), + np.concatenate(teachers, axis=0) if teachers else None, + np.concatenate(valid_masks, axis=0) if valid_masks else None, + ) + + +def write_semantic_occupancy( + path: str | Path, + sample_uids: Sequence[str], + probability: np.ndarray, + *, + teacher: np.ndarray | None = None, + valid_mask: np.ndarray | None = None, +) -> str: + payload = encode_semantic_occupancy( + sample_uids, + probability, + teacher=teacher, + valid_mask=valid_mask, + ) + output = Path(path) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_bytes(payload) + return hashlib.sha256(payload).hexdigest() diff --git a/Platform/pipelines/trajectory_visualization_tasks.py b/Platform/pipelines/trajectory_visualization_tasks.py index b421574f5..75cc55e1b 100644 --- a/Platform/pipelines/trajectory_visualization_tasks.py +++ b/Platform/pipelines/trajectory_visualization_tasks.py @@ -30,7 +30,7 @@ def export_trajectory_report( dataset_manifest: FlyteFile, overlay_manifest: FlyteFile, selection_manifest: Optional[FlyteFile] = None, - scene_uids: List[str] = [], + scene_uids: Optional[List[str]] = None, seed_index: int = 0, camera_index: int = 0, max_frames_per_scene: int = 300, diff --git a/Platform/pipelines/workflows.py b/Platform/pipelines/workflows.py index c181cb91a..38fccea7a 100644 --- a/Platform/pipelines/workflows.py +++ b/Platform/pipelines/workflows.py @@ -8,14 +8,22 @@ MLflow: Training logs epoch metrics; evaluation logs final metrics and registry entries. Two experiments: imitation-learning and offline-rl. """ +from __future__ import annotations + import enum import functools +from pathlib import Path from flytekit import ( task, workflow, dynamic, map_task, Resources, Secret, BatchSize, ) from flytekit.types.file import FlyteFile from flytekit.types.directory import FlyteDirectory -from typing import Annotated, NamedTuple, List, Optional +from typing import NamedTuple, List, Optional + +try: + from typing import Annotated +except ImportError: # Python 3.8 in the pinned BEVFormer V2 runtime. + from typing_extensions import Annotated from data_processing.contract_versions import ( GEOMETRY_VERSION as _GEOM_V, @@ -24,6 +32,10 @@ SHARD_SCHEMA_VERSION as _SHARD_V, UID_SCHEMA_VERSION as _UID_V, ) +from data_processing.source_revisions import L2D_DATA_REVISION +from data_parsing.kit_scenes.temporal_contract import ( + kitscenes_temporal_contract, +) from Platform.pipelines.dataset_publication import DatasetPublication from Platform.pipelines.overlay_tasks import ( register_selected_overlay_checkpoint, @@ -35,7 +47,7 @@ import os as _os -ECR_PREFIX = _os.environ.get("ECR_PREFIX", "381491877296.dkr.ecr.us-west-2.amazonaws.com") +ECR_PREFIX = _os.environ.get("ECR_PREFIX", "registry.invalid") TRAINING_IMAGE = _os.environ.get( "AUTO_E2E_TRAINING_IMAGE", f"{ECR_PREFIX}/auto-e2e/training:latest", @@ -52,17 +64,24 @@ "AUTO_E2E_DATA_PREP_IMAGE", f"{ECR_PREFIX}/auto-e2e/data-prep:latest", ) +BEVFORMER_V2_IMAGE = _os.environ.get( + "AUTO_E2E_BEVFORMER_V2_IMAGE", + f"{ECR_PREFIX}/auto-e2e/bevformer-v2:latest", +) MLFLOW_URI = "http://mlflow.mlflow.svc.cluster.local:5000" DATASET_PACK_VERSION = "v2.2" +L2D_REACTIVE_DATASET_VERSION = "v3.0-reactive-v1" KITSCENES_NAVIGATION_DATASET_VERSION = "v3.3" +KITSCENES_BENCHMARK_DATASET_VERSION = "v3.3-benchmark-v3" BASELINE_TRAINING_OBJECTIVE_VERSION = "trajectory_imitation_v1" KITSCENES_NAVIGATION_OBJECTIVE_VERSION = ( "kitscenes_navigation_objective_v1" ) ROLLOUT_ALIGNED_OBJECTIVE_VERSION = "rollout_aligned_planner_v1" ROLLOUT_ALIGNED_CONTROL_OBJECTIVE_VERSION = "rollout_aligned_control_v1" -L2D_SOURCE_REVISION = "main" +SIMPLE_XY_IMITATION_OBJECTIVE_VERSION = "simple_xy_imitation_v1" +L2D_SOURCE_REVISION = L2D_DATA_REVISION KITSCENES_SOURCE_REVISION = "6fde0034446669e2ed7235e4c7fe323cd23d599d" # The per-sample S3 label cache is REMOVED (#121 §3.4): at full L2D it was ~10M @@ -173,11 +192,41 @@ def _large_shm_pod_template(): # --- Enums --- class Dataset(enum.Enum): + NUPLAN = "nuplan/nuplan-v1.1" L2D = "yaak-ai/L2D" KITSCENES = "KIT-MRT/KITScenes-Multimodal" NVIDIA_PHYSICAL_AI = "nvidia/PhysicalAI-Autonomous-Vehicles" +KITSCENES_TRAINING_SPLIT = "train" +KITSCENES_BENCHMARK_SPLITS = frozenset({"val", "overlap_train_val"}) + + +def _validate_kitscenes_data_role( + *, + data_role: str, + source_split: str, +) -> None: + """Keep held-out KITScenes scenes outside every training workflow.""" + if data_role == "training": + if source_split != KITSCENES_TRAINING_SPLIT: + raise ValueError( + "KITScenes training accepts only the official train split, " + f"got {source_split!r}" + ) + return + if data_role == "benchmark": + if source_split not in KITSCENES_BENCHMARK_SPLITS: + raise ValueError( + "KITScenes benchmark preparation accepts only val and " + f"overlap_train_val, got {source_split!r}" + ) + return + raise ValueError( + f"unsupported KITScenes data_role {data_role!r}" + ) + + class Backbone(enum.Enum): SWIN_V2_TINY = "swin_v2_tiny" CONVNEXT_V2_TINY = "conv_next_v2_tiny" @@ -201,10 +250,31 @@ def _row_decode_worker_count(dataset: Dataset, row_count: int) -> int: """Bound row decoders by each parser's per-process memory footprint.""" # Each KITScenes child reparses the scene's Lanelet2 map and calibration. # Large scenes exceeded the 64 GiB pod limit with the generic 16-worker cap. - max_workers = 2 if dataset == Dataset.KITSCENES else 16 + # Each L2D child owns video decoders for seven streams. Four workers retain + # useful decode parallelism without approaching the 64 GiB pod limit. + max_workers = 2 if dataset == Dataset.KITSCENES else 4 return max(1, min(max_workers, row_count)) +def _use_parent_assembly_pack( + dataset: Dataset, + *, + has_samples: bool, + world_model: bool, + reactive_targets: bool, +) -> bool: + """Select the memory-bounded row-decode path for structured targets.""" + return ( + dataset != Dataset.NVIDIA_PHYSICAL_AI + and has_samples + and ( + world_model + or dataset == Dataset.KITSCENES + or reactive_targets + ) + ) + + # NOTE: view fusion is no longer selectable. The reactive-refactor (PR #94) # removed concat/cross_attn and hardcoded BEV fusion inside ReactiveE2E, and # dropped the `fusion_mode` argument from AutoE2E.__init__. We keep the string @@ -228,6 +298,18 @@ def _row_decode_worker_count(dataset: Dataset, row_count: int) -> int: predictions=FlyteFile, report=FlyteFile, ) +KITScenesBenchmarkManifestOutput = NamedTuple( + "KITScenesBenchmarkManifestOutput", + manifest=FlyteFile, + manifest_sha256=str, +) +KITScenesBenchmarkPreparationOutput = NamedTuple( + "KITScenesBenchmarkPreparationOutput", + val_shards=List[FlyteDirectory], + overlap_shards=List[FlyteDirectory], + manifest=FlyteFile, + manifest_sha256=str, +) ReconstructionAuditOutput = NamedTuple( "ReconstructionAuditOutput", thresholds_pass=bool, @@ -236,6 +318,51 @@ def _row_decode_worker_count(dataset: Dataset, row_count: int) -> int: report=FlyteFile, records=FlyteFile, ) +ReactiveTrainingProgramOutput = NamedTuple( + "ReactiveTrainingProgramOutput", + stage_a_checkpoint=FlyteFile, + stage_a_metadata=FlyteFile, + stage_b_checkpoint=FlyteFile, + stage_b_metadata=FlyteFile, + retention_report=FlyteFile, + retention_report_sha256=str, +) +ReactiveRetentionOutput = NamedTuple( + "ReactiveRetentionOutput", + report=FlyteFile, + report_sha256=str, +) +ReactiveBenchmarkProgramOutput = NamedTuple( + "ReactiveBenchmarkProgramOutput", + stage_a_ade_3s=float, + stage_a_fde_3s=float, + stage_a_ade_5s=float, + stage_a_fde_5s=float, + stage_a_predictions=FlyteFile, + stage_a_report=FlyteFile, + stage_b_ade_3s=float, + stage_b_fde_3s=float, + stage_b_ade_5s=float, + stage_b_fde_5s=float, + stage_b_predictions=FlyteFile, + stage_b_report=FlyteFile, +) +SemanticOccupancyPrecomputeOutput = NamedTuple( + "SemanticOccupancyPrecomputeOutput", + manifest_key=str, + manifest_sha256=str, + checkpoint_sha256=str, + shard_count=int, + sample_count=int, +) +NuPlanRawSnapshotOutput = NamedTuple( + "NuPlanRawSnapshotOutput", + manifest=FlyteFile, + manifest_sha256=str, + snapshot_prefix=str, + archive_count=int, + total_size_bytes=int, +) # wf_create_dataset returns just the ready-to-train WebDataset shards (train_il # reads reasoning supervision from in-shard reasoning.json members). The # versioned reasoning-label artifact persists independently in S3 (the @@ -1539,6 +1666,14 @@ def _reasoning_label_indices(ds, label_stride: int) -> List[int]: return sorted(selected) +def _packed_episode_count( + episodes: int, + group_ids: Optional[List[str]], +) -> int: + """Return the exact source-group count represented by one packed shard.""" + return len(group_ids) if group_ids is not None else episodes + + # ============================================================ # Task: Resolve the immutable fan-out inventory # ============================================================ @@ -1562,6 +1697,7 @@ def plan_fanout_partitions( max_partitions: int, max_missing_scenes: int = 1, split: str = "train", + data_role: str = "training", ) -> List[List[str]]: """Resolve source groups once and return deterministic mapped-task inputs. @@ -1592,16 +1728,15 @@ def plan_fanout_partitions( token = os.environ.get("HF_TOKEN", "") if dataset == Dataset.KITSCENES: + _validate_kitscenes_data_role( + data_role=data_role, + source_split=split, + ) if source_revision != KITSCENES_SOURCE_REVISION: raise ValueError( "KITScenes source_revision must match the audited pinned " f"revision {KITSCENES_SOURCE_REVISION}, got {source_revision!r}" ) - if split != "train": - raise ValueError( - "The full training fan-out currently accepts only the official " - f"KITScenes train split, got {split!r}" - ) if partition_size != 1: raise ValueError( "KITScenes requires partition_size=1 because calibration and " @@ -1630,10 +1765,14 @@ def plan_fanout_partitions( + json.dumps(inventory.metadata(), sort_keys=True) ) elif dataset == Dataset.L2D: + if data_role != "training" or split != "train": + raise ValueError( + "L2D fan-out supports only data_role='training', split='train'" + ) if source_revision != L2D_SOURCE_REVISION: raise ValueError( - "L2D currently supports only revision='main' because the v3.0 " - f"tag is stale; got {source_revision!r}" + "L2D requires the audited source revision " + f"{L2D_SOURCE_REVISION}; got {source_revision!r}" ) if episodes == 0 or start_ep >= 0: try: @@ -1653,6 +1792,11 @@ def plan_fanout_partitions( total = episodes group_ids = [str(index) for index in range(total)] else: + if data_role != "training" or split != "train": + raise ValueError( + "non-KITScenes fan-out supports only " + "data_role='training', split='train'" + ) raise NotImplementedError( "NVIDIA PhysicalAI fan-out remains deferred; use the existing " "single-dataset workflow for that source." @@ -1726,6 +1870,8 @@ def data_ingest( source_revision: str = L2D_SOURCE_REVISION, episodes: int = 3, group_ids: Optional[List[str]] = None, + source_split: str = "train", + data_role: str = "training", ) -> Annotated[FlyteDirectory, BatchSize(4)]: """Download raw dataset from HuggingFace (lerobot for L2D, physical_ai_av for NVIDIA). @@ -1757,6 +1903,10 @@ def data_ingest( shutil.rmtree(out_dir) if dataset == Dataset.KITSCENES: + _validate_kitscenes_data_role( + data_role=data_role, + source_split=source_split, + ) if source_revision != KITSCENES_SOURCE_REVISION: raise ValueError( "KITScenes ingest requires pinned source revision " @@ -1775,22 +1925,30 @@ def data_ingest( if group_ids is None: inventory = resolve_inventory( downloader.archives, - split="train", + split=source_split, source_revision=source_revision, - max_missing_scenes=1, + max_missing_scenes=( + 1 if data_role == "training" else 0 + ), ) scene_ids = list(inventory.selected_scene_ids) if episodes > 0: scene_ids = scene_ids[:episodes] else: scene_ids = [str(scene_id) for scene_id in group_ids] - downloader.download(scene_ids, expected_split="train") + downloader.download(scene_ids, expected_split=source_split) print( f"Ingested {dataset.value}@{source_revision}: " - f"{len(scene_ids)} scenes -> {out_dir}" + f"{len(scene_ids)} {source_split} scenes -> {out_dir}" ) return FlyteDirectory(out_dir) + if source_split != "train" or data_role != "training": + raise ValueError( + "non-KITScenes ingest supports only " + "data_role='training', source_split='train'" + ) + if dataset == Dataset.NVIDIA_PHYSICAL_AI: # NVIDIA PhysicalAI-AV: download via physical_ai_av SDK + unpack into the # parser layout (camera//, labels/egomotion/) that NvidiaAVDataset reads. @@ -1879,16 +2037,18 @@ def data_ingest( from huggingface_hub import hf_hub_download from concurrent.futures import ThreadPoolExecutor, as_completed import time - # revision="main" — lerobot 0.5.0 defaults to CODEBASE_VERSION="v3.0", but + # The audited commit resolves the active branch content. lerobot 0.5.0 + # defaults to CODEBASE_VERSION="v3.0", but # yaak-ai/L2D's v3.0 TAG points to a stale/broken snapshot (tasks.parquet # is 1485 bytes / 1 row at v3.0 vs 135484 bytes / 4219 rows on main; # episodes/data parquets are ~20% smaller too). Reading v3.0 causes # downstream KeyError in _absolute_to_relative_idx and IndexError in - # iloc[task_idx]. Pin to main so we always get the live L2D revision. + # iloc[task_idx]. Pin the audited main commit so later branch movement + # cannot change an experiment. if source_revision != L2D_SOURCE_REVISION: raise ValueError( - "L2D ingest supports revision='main' only because its v3.0 tag is " - f"stale; got {source_revision!r}" + "L2D ingest requires the audited source revision " + f"{L2D_SOURCE_REVISION}; got {source_revision!r}" ) _meta = LeRobotDatasetMetadata( repo_id=dataset.value, @@ -2004,8 +2164,9 @@ def _one_file(rel_path: str, attempt_i: int) -> tuple[str, bool, str]: @task( container_image=DATA_PREP_IMAGE, pod_template=_data_prep_pod_template(), - # Process-parallel pack workers use the pod's available cores for camera - # decode/JPEG. The deduplicated WM path decodes each physical row once. + # Process-parallel camera workers decode/JPEG each physical row once for WM, + # KITScenes, and Reactive target packs. L2D uses four decoder processes so + # each process can own its video readers without exceeding the pod limit. # KITScenes one-scene partitions use the same schedulable Guaranteed profile # as ingest. The raw scene plus deduplicated 256px camera pool stays below # the default NodeClass's allocatable ephemeral storage. @@ -2039,6 +2200,10 @@ def data_processing( reasoning_labels: Optional[FlyteDirectory] = None, group_ids: Optional[List[str]] = None, expected_reasoning_label_count: Optional[int] = None, + reactive_targets: bool = False, + osm_graph_snapshot: Optional[FlyteFile] = None, + source_split: str = "train", + data_role: str = "training", ) -> Annotated[FlyteDirectory, BatchSize(4)]: """Pre-extract aligned frames + egomotion → WebDataset shards. @@ -2058,6 +2223,7 @@ def data_processing( (num_frames/stride) matches the online dataset so shards and on-the-fly windows are identical. """ + import hashlib import os import io import json @@ -2073,9 +2239,56 @@ def data_processing( raise ValueError( "expected_reasoning_label_count requires reasoning_labels" ) + if reactive_targets and dataset == Dataset.L2D: + if osm_graph_snapshot is None: + raise ValueError( + "L2D reactive targets require a pinned OSM graph snapshot" + ) + elif osm_graph_snapshot is not None: + raise ValueError( + "osm_graph_snapshot is supported only for L2D reactive targets" + ) + if reactive_targets and dataset not in { + Dataset.L2D, + Dataset.KITSCENES, + }: + raise ValueError( + "generic data_processing supports reactive targets only for " + "L2D and KITScenes; nuPlan uses its scenario adapter" + ) + if dataset == Dataset.NUPLAN: + raise ValueError( + "nuPlan cannot use the LeRobot/KITScenes packer; provide shards " + "produced by the nuPlan scenario adapter" + ) + if dataset == Dataset.KITSCENES: + _validate_kitscenes_data_role( + data_role=data_role, + source_split=source_split, + ) + elif source_split != "train" or data_role != "training": + raise ValueError( + "non-KITScenes processing supports only " + "data_role='training', source_split='train'" + ) + benchmark_protocol = ( + dataset == Dataset.KITSCENES and data_role == "benchmark" + ) raw_path = raw_data.download() print(f"Processing raw data from: {raw_path} (dataset={dataset.value})") + osm_graph_snapshot_path = ( + osm_graph_snapshot.download() + if osm_graph_snapshot is not None + else None + ) + osm_snapshot = None + if osm_graph_snapshot_path is not None: + from data_parsing.l2d import load_l2d_osm_graph_snapshot + + osm_snapshot = load_l2d_osm_graph_snapshot( + osm_graph_snapshot_path + ) # Reasoning labels present ⇒ this is a full-loss run, and the JEPA/world-model # loss needs the WM window (future frames) packed — so force WM on. Note the @@ -2113,8 +2326,8 @@ def data_processing( else: if dataset == Dataset.L2D and source_revision != L2D_SOURCE_REVISION: raise ValueError( - "L2D pack supports revision='main' only because its v3.0 tag is " - f"stale; got {source_revision!r}" + "L2D pack requires the audited source revision " + f"{L2D_SOURCE_REVISION}; got {source_revision!r}" ) ep_list = ([int(g) for g in group_ids] if group_ids is not None else (list(range(episodes)) if episodes > 0 else None)) @@ -2136,19 +2349,25 @@ def data_processing( from data_parsing.kit_scenes import KitScenesDataset ds = KitScenesDataset( data_root=raw_path, - split="train", + split=source_split, scene_ids=ep_list, image_size=image_size, include_world_model_windows=world_model, include_navigation=False, + benchmark_protocol=benchmark_protocol, ) else: from data_parsing.l2d import L2DDataset # World-Model windows (#16/#13) are only produced when requested, so the # imitation-only path stays cheap (no extra frame decode). root=raw_path: # read the partition's materialized raw, don't re-hit HF. - ds = L2DDataset(repo_id=dataset.value, episodes=ep_list, - include_world_model_windows=world_model, root=raw_path) + ds = L2DDataset( + repo_id=dataset.value, + revision=source_revision, + episodes=ep_list, + include_world_model_windows=world_model, + root=raw_path, + ) n_samples = len(ds) idx_iter = range(n_samples) except ValueError as e: @@ -2168,7 +2387,6 @@ def data_processing( labels_by_id = {} _record_to_json = None if reasoning_labels is not None: - from pathlib import Path from data_processing.reasoning_label_generation.targets import ( load_records_by_sample_id, record_to_json, ) @@ -2265,6 +2483,8 @@ def data_processing( shard_idx = 0 shard_names: list[str] = [] + shard_sample_counts: dict[str, int] = {} + current_shard_name: str | None = None sample_count = 0 reasoning_label_count = 0 joined_reasoning_ids: set[str] = set() @@ -2291,12 +2511,14 @@ def _write_pool(frame_id, blob): pool_frames_written += 1 def open_new_shard(): - nonlocal current_tar, shard_idx + nonlocal current_tar, current_shard_name, shard_idx if current_tar: current_tar.close() shard_name = published_shard_name(group_ids, shard_idx) current_tar = tarfile.open(os.path.join(out_dir, shard_name), "w") shard_names.append(shard_name) + shard_sample_counts[shard_name] = 0 + current_shard_name = shard_name shard_idx += 1 # Decode+JPEG-encode happens in the pack workers (parallel_pack); the parent @@ -2316,11 +2538,15 @@ def _add_member(sample_key, suffix, blob): has_map = False has_wm = False navigation_artifact_summary = None + trajectory_xy_count = 0 + bev_segmentation_count = 0 + reactive_navigation_count = 0 - if ( - dataset != Dataset.NVIDIA_PHYSICAL_AI - and idx_list - and (world_model or dataset == Dataset.KITSCENES) + if _use_parent_assembly_pack( + dataset, + has_samples=bool(idx_list), + world_model=world_model, + reactive_targets=reactive_targets, ): # ── DECODE-DEDUP path: decode each UNIQUE physical row once ── # (#121 §3.4d) Previous approach decoded all 48 window frames per sample @@ -2334,14 +2560,23 @@ def _add_member(sample_key, suffix, blob): # reasoning JOIN) from the pool — zero video decode. print(f"Packing {len(idx_list)} samples, parent-assembly mode " f"(row-level camera workers, world_model={world_model})...") - row_init = (dataset.value, ep_list, raw_path, image_size) + row_init = ( + dataset.value, + ep_list, + raw_path, + image_size, + source_split, + source_revision, + benchmark_protocol, + ) # Pass A: unique rows. ds is still alive here (not yet deleted). all_rows: set = set() # Collect the current-frame row (offset 0 = cam_*.jpg) FIRST so it's # tracked even if window_rows raises. Do NOT catch IndexError from - # window_rows: enumeration excludes edge frames (margins 64/64 dominate - # WM 30/40), so a raise here means the invariant has broken and we MUST + # window_rows: enumeration excludes edge frames. Training uses 64/64 + # margins and the benchmark uses 40/50; both cover the WM 30/40 window. + # A raise here means the invariant has broken and we MUST # fail loudly rather than silently drop the sample's cam_*.jpg (which # would poison the shard: loader hits torch.stack([]) at train time). sample_cur_rows: dict = {} # si -> (episode/scene, frame) current row @@ -2393,17 +2628,19 @@ def _add_member(sample_key, suffix, blob): from data_parsing.kit_scenes import KitScenesDataset ds_asm = KitScenesDataset( data_root=raw_path, - split="train", + split=source_split, scene_ids=ep_list, image_size=image_size, include_world_model_windows=False, include_navigation=True, source_revision=source_revision, + benchmark_protocol=benchmark_protocol, ) else: from data_parsing.l2d import L2DDataset ds_asm = L2DDataset( repo_id=dataset.value, + revision=source_revision, episodes=ep_list, include_world_model_windows=False, root=raw_path, @@ -2484,6 +2721,54 @@ def _add_member(sample_key, suffix, blob): "pose_current": pose_current, "gps_future": gps_future, })) + if ( + reactive_targets + and pose_current is not None + and gps_future is not None + ): + from data_processing.reactive_training_artifacts import ( + TRAJECTORY_XY_MEMBER, + encode_trajectory_xy, + wgs84_future_to_ego_xy, + ) + + trajectory_xy, trajectory_valid = ( + wgs84_future_to_ego_xy( + gps_future, + current_latitude_deg=float( + pose_current["latitude_deg"] + ), + current_longitude_deg=float( + pose_current["longitude_deg"] + ), + heading_deg_cw_from_north=float( + pose_current[ + "heading_deg_cw_from_north" + ] + ), + ) + ) + members[TRAJECTORY_XY_MEMBER] = encode_trajectory_xy( + trajectory_xy, + trajectory_valid, + ) + if dataset == Dataset.L2D and osm_snapshot is not None: + if pose_current is None: + raise ValueError( + "L2D reactive targets require the current GPS pose" + ) + from data_parsing.l2d import ( + l2d_reactive_navigation_members, + ) + + members.update( + l2d_reactive_navigation_members( + osm_snapshot, + ds_asm.route_waypoints_for(si), + pose_current, + ) + ) + has_map = True members["meta.json"] = json.dumps({ "idx": si, "dataset": dataset.value, "sample_uid": uid, "split_group_uid": split_group, @@ -2494,6 +2779,13 @@ def _add_member(sample_key, suffix, blob): for suffix, blob in members.items(): _add_member(uid, suffix, blob) + trajectory_xy_count += int("trajectory_xy.npz" in members) + bev_segmentation_count += int( + "bev_segmentation.npz" in members + ) + reactive_navigation_count += int( + "navigation_meta.json" in members + ) if _record_to_json is not None: record = labels_by_id.get(uid) if record is not None: @@ -2502,6 +2794,8 @@ def _add_member(sample_key, suffix, blob): reasoning_label_count += 1 joined_reasoning_ids.add(uid) sample_count += 1 + assert current_shard_name is not None + shard_sample_counts[current_shard_name] += 1 else: # ── Legacy path (imitation-only L2D, NVIDIA, or empty partition) ── @@ -2510,7 +2804,15 @@ def _add_member(sample_key, suffix, blob): pack_workers = max(1, min(max_workers_cap, len(idx_list))) print(f"Packing {len(idx_list)} samples, legacy mode " f"(world_model={world_model}, per-sample decode)...") - pack_init = (dataset.value, ep_list, raw_path, image_size, world_model, calib_bytes) + pack_init = ( + dataset.value, + ep_list, + raw_path, + image_size, + world_model, + calib_bytes, + osm_graph_snapshot_path, + ) del ds with ProcessPoolExecutor(max_workers=pack_workers, mp_context=ctx, initializer=parallel_pack.init_pack_worker, @@ -2529,6 +2831,15 @@ def _add_member(sample_key, suffix, blob): or "map_semantic.npz" in members ) has_wm = has_wm or ("window_index.json" in members) + trajectory_xy_count += int( + "trajectory_xy.npz" in members + ) + bev_segmentation_count += int( + "bev_segmentation.npz" in members + ) + reactive_navigation_count += int( + "navigation_meta.json" in members + ) if _record_to_json is not None: record = labels_by_id.get(sample_key) if record is not None: @@ -2537,9 +2848,27 @@ def _add_member(sample_key, suffix, blob): reasoning_label_count += 1 joined_reasoning_ids.add(sample_key) sample_count += 1 + assert current_shard_name is not None + shard_sample_counts[current_shard_name] += 1 if current_tar: current_tar.close() + shard_sha256 = { + name: hashlib.sha256( + Path(out_dir, name).read_bytes() + ).hexdigest() + for name in shard_names + } + + if ( + reactive_targets + and sample_count + and reactive_navigation_count != sample_count + ): + raise ValueError( + "reactive target packing was incomplete: " + f"{reactive_navigation_count}/{sample_count} samples" + ) if expected_reasoning_label_count is not None: unjoined_ids = set(labels_by_id) - joined_reasoning_ids @@ -2566,31 +2895,66 @@ def _add_member(sample_key, suffix, blob): GPS_SCHEMA_VERSION, POSE_SCHEMA_VERSION, ) - from navigation.geometry import DEFAULT_NAVIGATION_GEOMETRY + from data_processing.reactive_training_artifacts import ( + BEV_SEGMENTATION_ARTIFACT_VERSION, + BEV_SEGMENTATION_CLASSES, + REACTIVE_NAVIGATION_ARTIFACT_VERSION, + TRAJECTORY_XY_ARTIFACT_VERSION, + ) + from navigation.geometry import ( + AUTOE2E_NAVIGATION_GEOMETRY, + DEFAULT_NAVIGATION_GEOMETRY, + ) from navigation.supervision import ( ROUTE_SUPERVISION_ARTIFACT_VERSION, ) manifest = {"total_samples": sample_count, "shards": shard_idx, "shard_names": shard_names, + "shard_sample_counts": shard_sample_counts, + "shard_sha256": shard_sha256, "partition_id": partition_id or None, "hz": hz, "image_size": image_size, "dataset": dataset.value, "source_revision": source_revision, + "source_split": source_split, + "data_role": data_role, "dataset_version": dataset_version, - "episodes": episodes, + "episodes": _packed_episode_count(episodes, group_ids), + "temporal_sampling": ( + kitscenes_temporal_contract( + benchmark_protocol=benchmark_protocol, + ) + if dataset == Dataset.KITSCENES + else None + ), + "reactive_targets_requested": reactive_targets, "contracts": contract_versions(), # num_views = real cameras only; the map view is stored under a # separate map.jpg key and is NOT counted here (#77). "num_views": num_views if sample_count else 0, "has_map": bool(sample_count) and has_map, - "has_navigation": ( + "has_navigation": bool(sample_count) and ( + navigation_artifact_summary is not None + or reactive_navigation_count == sample_count + ), + "has_reactive_navigation": ( bool(sample_count) - and navigation_artifact_summary is not None + and reactive_navigation_count == sample_count + ), + "reactive_navigation_count": reactive_navigation_count, + "reactive_navigation_version": ( + REACTIVE_NAVIGATION_ARTIFACT_VERSION + if reactive_navigation_count + else None ), "has_route_supervision": ( bool(sample_count) and navigation_artifact_summary is not None ), + "has_route_reconstruction": ( + bool(sample_count) + and reactive_navigation_count == sample_count + ), "route_supervision_version": ( ROUTE_SUPERVISION_ARTIFACT_VERSION if ( @@ -2600,15 +2964,62 @@ def _add_member(sample_key, suffix, blob): else None ), "navigation": navigation_artifact_summary, + "navigation_source": ( + { + "type": "pinned_osm_graph", + "sha256": osm_snapshot.source_sha256, + "revision": osm_snapshot.source_revision, + "attribution": osm_snapshot.attribution, + } + if osm_snapshot is not None + else None + ), "navigation_geometry": ( - DEFAULT_NAVIGATION_GEOMETRY.contract() - if navigation_artifact_summary is not None + ( + AUTOE2E_NAVIGATION_GEOMETRY.contract() + if reactive_navigation_count + else DEFAULT_NAVIGATION_GEOMETRY.contract() + ) + if ( + navigation_artifact_summary is not None + or reactive_navigation_count + ) else None ), "map_context_channels": ( - 14 if navigation_artifact_summary is not None else 3 + 14 + if ( + navigation_artifact_summary is not None + or reactive_navigation_count + ) + else 3 ), "route_channels": 2, + "has_trajectory_xy": ( + bool(sample_count) + and trajectory_xy_count == sample_count + ), + "trajectory_xy_count": trajectory_xy_count, + "trajectory_xy_version": ( + TRAJECTORY_XY_ARTIFACT_VERSION + if trajectory_xy_count + else None + ), + "has_bev_segmentation": ( + bool(sample_count) + and bev_segmentation_count == sample_count + ), + "bev_segmentation_count": bev_segmentation_count, + "bev_segmentation_version": ( + BEV_SEGMENTATION_ARTIFACT_VERSION + if bev_segmentation_count + else None + ), + "bev_segmentation_classes": ( + list(BEV_SEGMENTATION_CLASSES) + if bev_segmentation_count + else None + ), # World-Model windows present when packed (enables JEPA training). "has_world_model": bool(sample_count) and has_wm, "has_reasoning_labels": reasoning_label_count > 0, @@ -2887,8 +3298,8 @@ def generate_reasoning_labels( else: if dataset == Dataset.L2D and source_revision != L2D_SOURCE_REVISION: raise ValueError( - "L2D labeling supports revision='main' only because its v3.0 " - f"tag is stale; got {source_revision!r}" + "L2D labeling requires the audited source revision " + f"{L2D_SOURCE_REVISION}; got {source_revision!r}" ) ep_list = ([int(g) for g in group_ids] if group_ids is not None else (list(range(episodes)) if episodes > 0 else None)) @@ -2920,8 +3331,13 @@ def generate_reasoning_labels( else: from data_parsing.l2d import L2DDataset # root=raw_path: read the partition's materialized raw, don't re-hit HF. - ds = L2DDataset(repo_id=dataset.value, episodes=ep_list, - reasoning_clip_only=True, root=raw_path) + ds = L2DDataset( + repo_id=dataset.value, + revision=source_revision, + episodes=ep_list, + reasoning_clip_only=True, + root=raw_path, + ) n_samples = len(ds) label_indices = _reasoning_label_indices(ds, label_stride) except ValueError as e: @@ -6149,74 +6565,2117 @@ def _gradient_list_norm(gradients): # ============================================================ -# Task: Offline RL +# Task: authorized nuPlan source -> immutable raw snapshot # ============================================================ @task( - container_image=OFFLINE_RL_IMAGE, - # requests == limits (Guaranteed QoS). - requests=Resources(cpu="4", mem="16Gi", gpu="1"), - limits=Resources(cpu="4", mem="16Gi", gpu="1"), + container_image=DATA_PREP_IMAGE, + pod_template=_data_prep_pod_template(), + requests=Resources( + cpu="2", + mem="4Gi", + ephemeral_storage="4Gi", + ), + limits=Resources( + cpu="2", + mem="4Gi", + ephemeral_storage="4Gi", + ), + retries=2, ) -def train_offline_rl( - pretrained: FlyteFile, - shards: List[FlyteDirectory], - il_metadata: FlyteFile, - dataset: Dataset = Dataset.L2D, - epochs: int = 3, - tau: float = 0.7, - beta: float = 3.0, -) -> TrainOutput: - """Offline RL refinement of the IL checkpoint via advantage-weighted regression - against a frozen IL prior (AWR — not full IQL; no learned value network).""" - import os +def acquire_nuplan_archive( + source_manifest: FlyteFile, + archive_index: int, + datasets_bucket: str, + aws_region: str = "us-west-2", +) -> FlyteFile: + """Import one authorized nuPlan archive into an immutable S3 snapshot.""" import json - import torch - import numpy as np - from flytekit import current_context - - ckpt_path = pretrained.download() - il_meta = json.load(open(il_metadata.download())) - ctx = current_context() - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - - print(f"Offline RL (AWR, frozen prior): epochs={epochs} beta={beta}") + import tempfile + from contextlib import closing + from pathlib import Path + from urllib.error import HTTPError, URLError + from urllib.parse import urlsplit + from urllib.request import HTTPRedirectHandler, Request, build_opener - # Load IL model - from model_components.auto_e2e import AutoE2E - from data_parsing.pre_extracted import make_pre_extracted_loader + import boto3 + from botocore import UNSIGNED + from botocore.config import Config + from botocore.exceptions import ClientError + + from Platform.pipelines.nuplan_acquisition import ( + ARCHIVE_RECEIPT_SCHEMA_VERSION, + archive_object_key, + archive_receipt_key, + canonical_json_bytes, + copy_s3_object_multipart, + digest_stream, + load_source_manifest_bytes, + official_nuplan_open_data_region, + upload_https_stream_multipart, + validate_archive_digest, + validate_public_https_uri, + validate_s3_source_head, + ) - import copy + if ( + not datasets_bucket + or datasets_bucket.startswith("s3://") + or "/" in datasets_bucket + ): + raise ValueError("datasets_bucket must be one S3 bucket name") + if archive_index < 0: + raise ValueError("archive_index must be non-negative") + + source_bytes = Path(source_manifest.download()).read_bytes() + manifest, source_contract_sha256 = load_source_manifest_bytes(source_bytes) + if archive_index >= len(manifest["archives"]): + raise IndexError( + f"archive_index {archive_index} is outside " + f"{len(manifest['archives'])} source archives" + ) + archive = manifest["archives"][archive_index] + parsed_source = urlsplit(archive["source_uri"]) + object_key = archive_object_key(manifest, archive) + receipt_key = archive_receipt_key(manifest, archive) + s3 = boto3.client("s3", region_name=aws_region) + + def receipt_output(payload: bytes) -> FlyteFile: + path = Path(tempfile.mkdtemp(prefix="nuplan-archive-receipt-")) + output = path / "receipt.json" + output.write_bytes(payload) + return FlyteFile(str(output)) - ckpt = torch.load( - ckpt_path, - map_location=device, - weights_only=False, - ) - config = ckpt["config"] - from training.dataset_policy import ( - adapt_egomotion_history, - training_policy_from_config, - ) + try: + response = s3.get_object( + Bucket=datasets_bucket, + Key=receipt_key, + ) + except ClientError as error: + if error.response.get("Error", {}).get("Code") not in { + "404", + "NoSuchKey", + }: + raise + else: + with closing(response["Body"]) as receipt_stream: + receipt_bytes = receipt_stream.read() + receipt = json.loads(receipt_bytes) + if ( + receipt.get("schema_version") + != ARCHIVE_RECEIPT_SCHEMA_VERSION + or receipt.get("archive_id") != archive["archive_id"] + or receipt.get("source_contract_sha256") + != source_contract_sha256 + or receipt.get("object_uri") + != f"s3://{datasets_bucket}/{object_key}" + ): + raise ValueError( + f"existing nuPlan receipt conflicts with {archive['archive_id']!r}" + ) + head_arguments = { + "Bucket": datasets_bucket, + "Key": object_key, + } + if receipt.get("transfer_mode") == "s3_server_side_multipart_copy": + head_arguments["ChecksumMode"] = "ENABLED" + head = s3.head_object(**head_arguments) + if int(head["ContentLength"]) != int(receipt["size_bytes"]): + raise ValueError( + f"existing nuPlan object size differs from receipt: {object_key}" + ) + if ( + receipt.get("checksum_crc64nvme") + and head.get("ChecksumCRC64NVME") + != receipt["checksum_crc64nvme"] + ): + raise ValueError( + "existing nuPlan object checksum differs from receipt: " + f"{object_key}" + ) + return receipt_output(receipt_bytes) - training_policy = training_policy_from_config( - config, - dataset.value, - ) - if training_policy.validation_strategy != "hash_buckets": - raise ValueError( - "offline RL does not yet support an exact KITScenes train/holdout " - "partition; refusing to train on one shard or leak validation scenes" + upload = None + try: + head = s3.head_object( + Bucket=datasets_bucket, + Key=object_key, + **( + {"ChecksumMode": "ENABLED"} + if parsed_source.scheme == "s3" + else {} + ), + ) + except ClientError as error: + if error.response.get("Error", {}).get("Code") not in { + "404", + "NoSuchKey", + }: + raise + else: + if int(head["ContentLength"]) != int(archive["expected_size_bytes"]): + raise ValueError( + f"existing nuPlan object has the wrong size: {object_key}" + ) + expected_metadata = { + "archive-id": archive["archive_id"], + "snapshot-id": manifest["snapshot_id"], + "source-contract-sha256": source_contract_sha256, + **( + {"source-etag": archive["expected_etag"]} + if archive["expected_etag"] + else {} + ), + } + actual_metadata = head.get("Metadata", {}) + if any( + actual_metadata.get(key) != value + for key, value in expected_metadata.items() + ): + raise ValueError( + "existing nuPlan object metadata differs from its source " + f"contract: {object_key}" + ) + if parsed_source.scheme == "s3": + checksum = head.get("ChecksumCRC64NVME") + if not isinstance(checksum, str) or not checksum: + raise ValueError( + "existing server-side copied nuPlan object lacks " + f"CRC64NVME: {object_key}" + ) + upload = { + "checksum_crc64nvme": checksum, + "destination_etag": str(head["ETag"]).strip('"').lower(), + "md5": "", + "sha256": "", + "size_bytes": int(head["ContentLength"]), + "source_etag": archive["expected_etag"], + "transfer_mode": "s3_server_side_multipart_copy", + } + else: + existing_object = s3.get_object( + Bucket=datasets_bucket, + Key=object_key, + ) + with closing(existing_object["Body"]) as existing_stream: + upload = digest_stream(existing_stream) + validate_archive_digest( + upload, + expected_size_bytes=archive["expected_size_bytes"], + expected_sha256=archive["expected_sha256"], + expected_md5=archive["expected_md5"], + label=object_key, + ) + upload.update({ + "checksum_crc64nvme": "", + "destination_etag": str(head["ETag"]).strip('"').lower(), + "source_etag": "", + "transfer_mode": "https_stream_hash", + }) + print( + "Recovered nuPlan archive receipt from existing verified object " + f"id={archive['archive_id']}" ) - shard_dir = _select_shard_dir(shards, dataset) - model = AutoE2E(**_model_kwargs(config)).to(device) - model.load_state_dict(ckpt["model_state_dict"]) - # FROZEN behavior prior = the IL checkpoint at t=0, kept fixed. The advantage - # must be measured against a policy that does NOT move with the one being - # trained; using the LIVE model for both terms makes advantage identically 0 - # (a no-op that silently reduces to plain BC). This frozen prior gives a real - # signal: "does the fine-tuned policy beat the IL prior on this sample?". - baseline_model = copy.deepcopy(model).to(device).eval() + if upload is None: + if parsed_source.scheme == "s3": + source_bucket = parsed_source.netloc + source_key = parsed_source.path.lstrip("/") + open_data_region = official_nuplan_open_data_region( + source_bucket, + source_key, + ) + if open_data_region is None: + source_s3 = boto3.client("s3") + else: + source_s3 = boto3.client( + "s3", + region_name=open_data_region, + config=Config(signature_version=UNSIGNED), + ) + source_head = source_s3.head_object( + Bucket=source_bucket, + Key=source_key, + ) + validate_s3_source_head(source_head, archive) + upload = copy_s3_object_multipart( + s3_client=s3, + source_bucket=source_bucket, + source_key=source_key, + source_etag=archive["expected_etag"], + destination_bucket=datasets_bucket, + destination_key=object_key, + metadata={ + "archive-id": archive["archive_id"], + "snapshot-id": manifest["snapshot_id"], + "source-contract-sha256": source_contract_sha256, + "source-etag": archive["expected_etag"], + }, + expected_size_bytes=archive["expected_size_bytes"], + ) + else: + validate_public_https_uri(archive["source_uri"]) + + class PublicHTTPSRedirectHandler(HTTPRedirectHandler): + def redirect_request( + self, + request, + file_pointer, + code, + message, + headers, + new_url, + ): + validate_public_https_uri(new_url) + return super().redirect_request( + request, + file_pointer, + code, + message, + headers, + new_url, + ) + + request = Request( + archive["source_uri"], + headers={ + "Accept-Encoding": "identity", + "User-Agent": "auto-e2e-nuplan-acquisition/1", + }, + ) + try: + source_response = build_opener( + PublicHTTPSRedirectHandler() + ).open( + request, + timeout=120, + ) + except HTTPError as error: + raise RuntimeError( + "authorized HTTPS source returned " + f"status={error.code} archive_id={archive['archive_id']}" + ) from None + except URLError as error: + raise RuntimeError( + "authorized HTTPS source connection failed " + f"archive_id={archive['archive_id']} " + f"reason_type={type(error.reason).__name__}" + ) from None + with closing(source_response) as source_stream: + upload = upload_https_stream_multipart( + s3_client=s3, + stream=source_stream, + bucket=datasets_bucket, + key=object_key, + metadata={ + "archive-id": archive["archive_id"], + "snapshot-id": manifest["snapshot_id"], + "source-contract-sha256": source_contract_sha256, + }, + expected_size_bytes=archive["expected_size_bytes"], + expected_sha256=archive["expected_sha256"], + expected_md5=archive["expected_md5"], + ) + upload.update({ + "checksum_crc64nvme": "", + "source_etag": "", + "transfer_mode": "https_stream_hash", + }) + + head = s3.head_object( + Bucket=datasets_bucket, + Key=object_key, + **( + {"ChecksumMode": "ENABLED"} + if upload["transfer_mode"] == "s3_server_side_multipart_copy" + else {} + ), + ) + if int(head["ContentLength"]) != int(upload["size_bytes"]): + raise ValueError( + f"uploaded nuPlan archive size differs after completion: {object_key}" + ) + upload.setdefault( + "destination_etag", + str(head["ETag"]).strip('"').lower(), + ) + receipt = { + "archive_id": archive["archive_id"], + "checksum_crc64nvme": upload.get("checksum_crc64nvme", ""), + "component": archive["component"], + "destination_etag": upload["destination_etag"], + "md5": upload["md5"], + "object_uri": f"s3://{datasets_bucket}/{object_key}", + "schema_version": ARCHIVE_RECEIPT_SCHEMA_VERSION, + "sha256": upload["sha256"], + "size_bytes": upload["size_bytes"], + "source_contract_sha256": source_contract_sha256, + "source_etag": upload.get("source_etag", ""), + "transfer_mode": upload["transfer_mode"], + } + receipt_bytes = canonical_json_bytes(receipt) + try: + s3.put_object( + Bucket=datasets_bucket, + Key=receipt_key, + Body=receipt_bytes, + ContentType="application/json", + IfNoneMatch="*", + ) + except ClientError as error: + if error.response.get("Error", {}).get("Code") not in { + "PreconditionFailed", + "412", + }: + raise + existing = s3.get_object( + Bucket=datasets_bucket, + Key=receipt_key, + )["Body"].read() + if existing != receipt_bytes: + raise ValueError( + f"concurrent nuPlan receipt differs for {archive['archive_id']!r}" + ) from error + print( + "Imported nuPlan archive " + f"id={archive['archive_id']} component={archive['component']} " + f"size_bytes={upload['size_bytes']} " + f"transfer_mode={upload['transfer_mode']} " + f"integrity={upload.get('checksum_crc64nvme') or upload['sha256']}" + ) + return receipt_output(receipt_bytes) + + +@task( + container_image=DATA_PREP_IMAGE, + pod_template=_data_prep_pod_template(), + requests=Resources(cpu="1", mem="2Gi", ephemeral_storage="2Gi"), + limits=Resources(cpu="1", mem="2Gi", ephemeral_storage="2Gi"), +) +def finalize_nuplan_raw_snapshot( + source_manifest: FlyteFile, + archive_receipts: List[FlyteFile], + datasets_bucket: str, + aws_region: str = "us-west-2", +) -> NuPlanRawSnapshotOutput: + """Publish the redacted canonical manifest after every archive is verified.""" + import json + from pathlib import Path + + import boto3 + from botocore.exceptions import ClientError + + from Platform.pipelines.nuplan_acquisition import ( + build_snapshot_manifest, + canonical_json_bytes, + load_source_manifest_bytes, + sha256_bytes, + snapshot_manifest_key, + snapshot_prefix, + ) + + source_bytes = Path(source_manifest.download()).read_bytes() + manifest, source_contract_sha256 = load_source_manifest_bytes(source_bytes) + receipts = [ + json.loads(Path(receipt.download()).read_text(encoding="utf-8")) + for receipt in archive_receipts + ] + snapshot = build_snapshot_manifest( + source_manifest=manifest, + source_contract_sha256=source_contract_sha256, + receipts=receipts, + ) + payload = canonical_json_bytes(snapshot) + payload_sha256 = sha256_bytes(payload) + key = snapshot_manifest_key(manifest) + s3 = boto3.client("s3", region_name=aws_region) + try: + s3.put_object( + Bucket=datasets_bucket, + Key=key, + Body=payload, + ContentType="application/json", + Metadata={"manifest-sha256": payload_sha256}, + IfNoneMatch="*", + ) + except ClientError as error: + if error.response.get("Error", {}).get("Code") not in { + "PreconditionFailed", + "412", + }: + raise + existing = s3.get_object(Bucket=datasets_bucket, Key=key)["Body"].read() + if existing != payload: + raise ValueError( + "existing nuPlan snapshot manifest differs for " + f"{manifest['snapshot_id']!r}" + ) from error + manifest_uri = f"s3://{datasets_bucket}/{key}" + print( + "Published nuPlan raw snapshot " + f"id={manifest['snapshot_id']} archives={len(receipts)} " + f"total_size_bytes={snapshot['total_size_bytes']} " + f"manifest_sha256={payload_sha256}" + ) + return NuPlanRawSnapshotOutput( + manifest=FlyteFile(manifest_uri), + manifest_sha256=payload_sha256, + snapshot_prefix=( + f"s3://{datasets_bucket}/{snapshot_prefix(manifest)}" + ), + archive_count=len(receipts), + total_size_bytes=int(snapshot["total_size_bytes"]), + ) + + +@dynamic( + container_image=DATA_PREP_IMAGE, + environment={"AUTO_E2E_DATA_PREP_IMAGE": DATA_PREP_IMAGE}, +) +def _acquire_nuplan_raw_snapshot( + source_manifest: FlyteFile, + datasets_bucket: str, + aws_region: str, + concurrency: int, +) -> NuPlanRawSnapshotOutput: + """Fan out authorized archive imports without exposing signed URLs.""" + from pathlib import Path + + from Platform.pipelines.nuplan_acquisition import load_source_manifest_bytes + + if concurrency <= 0: + raise ValueError("concurrency must be positive") + manifest, _ = load_source_manifest_bytes( + Path(source_manifest.download()).read_bytes() + ) + importer = map_task( + functools.partial( + acquire_nuplan_archive, + source_manifest=source_manifest, + datasets_bucket=datasets_bucket, + aws_region=aws_region, + ), + concurrency=concurrency, + ) + receipts = importer( + archive_index=list(range(len(manifest["archives"]))) + ) + return finalize_nuplan_raw_snapshot( + source_manifest=source_manifest, + archive_receipts=receipts, + datasets_bucket=datasets_bucket, + aws_region=aws_region, + ) + + +# ============================================================ +# Task: raw nuPlan -> immutable Reactive shards +# ============================================================ +@task( + container_image=DATA_PREP_IMAGE, + requests=Resources(cpu="8", mem="32Gi"), + limits=Resources(cpu="8", mem="32Gi"), +) +def pack_nuplan_reactive_dataset( + data_root: FlyteDirectory, + map_root: FlyteDirectory, + sensor_root: FlyteDirectory, + db_files: List[str], + source_revision: str, + map_version: str, + limit_total_scenarios: int = 0, + image_size: int = 256, + samples_per_shard: int = 1000, + max_rejection_fraction: float = 0.0, +) -> FlyteDirectory: + """Pack raw local nuPlan scenarios with camera, BEV, Route, and XY targets.""" + import os + import tempfile + from pathlib import Path + + from data_parsing.nuplan import pack_nuplan_reactive_scenarios + from nuplan.planning.scenario_builder.nuplan_db.nuplan_scenario_builder import ( + NuPlanScenarioBuilder, + ) + from nuplan.planning.scenario_builder.scenario_filter import ( + ScenarioFilter, + ) + from nuplan.planning.utils.multithreading.worker_sequential import ( + Sequential, + ) + + if not source_revision or not map_version: + raise ValueError("nuPlan source_revision and map_version are required") + if limit_total_scenarios < 0: + raise ValueError("limit_total_scenarios must be non-negative") + local_data = Path(data_root.download()).resolve() + local_map = Path(map_root.download()).resolve() + local_sensor = Path(sensor_root.download()).resolve() + for name, path in ( + ("data_root", local_data), + ("map_root", local_map), + ("sensor_root", local_sensor), + ): + if not path.is_dir(): + raise FileNotFoundError(f"nuPlan {name} is not a directory: {path}") + + resolved_db_files = [] + for relative in db_files: + candidate = (local_data / relative).resolve() + if local_data not in candidate.parents or candidate.suffix != ".db": + raise ValueError( + "nuPlan db_files must be relative .db children of data_root" + ) + if not candidate.is_file(): + raise FileNotFoundError(f"nuPlan DB is missing: {candidate}") + resolved_db_files.append(str(candidate)) + os.environ["NUPLAN_DATA_STORE"] = "local" + builder = NuPlanScenarioBuilder( + data_root=str(local_data), + map_root=str(local_map), + sensor_root=str(local_sensor), + db_files=resolved_db_files or None, + map_version=map_version, + include_cameras=True, + max_workers=1, + verbose=False, + ) + scenario_filter = ScenarioFilter( + scenario_types=None, + scenario_tokens=None, + log_names=None, + map_names=None, + num_scenarios_per_type=None, + limit_total_scenarios=( + limit_total_scenarios or None + ), + timestamp_threshold_s=None, + ego_displacement_minimum_m=None, + expand_scenarios=False, + remove_invalid_goals=True, + shuffle=False, + ) + scenarios = builder.get_scenarios( + scenario_filter, + Sequential(), + ) + output = Path(tempfile.mkdtemp(prefix="nuplan-reactive-shards-")) + pack_nuplan_reactive_scenarios( + scenarios, + output, + source_revision=source_revision, + map_version=map_version, + image_size=image_size, + samples_per_shard=samples_per_shard, + max_rejection_fraction=max_rejection_fraction, + ) + return FlyteDirectory(str(output)) + + +@task( + container_image=DATA_PREP_IMAGE, + requests=Resources(cpu="4", mem="16Gi"), + limits=Resources(cpu="4", mem="16Gi"), +) +def build_l2d_osm_graph_artifact( + source_pbf: FlyteFile, + source_revision: str, + source_date: str, + attribution: str = "OpenStreetMap contributors", +) -> FlyteFile: + """Convert one pinned regional OSM PBF into the canonical L2D graph.""" + import shutil + import tempfile + from pathlib import Path + + from data_parsing.l2d import build_l2d_osm_graph_snapshot + + if not source_revision or not source_date or not attribution: + raise ValueError("OSM provenance fields must not be empty") + downloaded = Path(source_pbf.download()) + output_directory = Path( + tempfile.mkdtemp(prefix="l2d-osm-graph-") + ) + source = downloaded + if source.suffixes[-2:] != [".osm", ".pbf"]: + source = output_directory / "source.osm.pbf" + shutil.copyfile(downloaded, source) + output = output_directory / "l2d-osm-graph.json" + build_l2d_osm_graph_snapshot( + source, + output, + source_revision=source_revision, + source_date=source_date, + attribution=attribution, + ) + return FlyteFile(str(output)) + + +# ============================================================ +# Task: Reactive nuPlan -> L2D multi-stage training +# ============================================================ +@task( + container_image=TRAINING_IMAGE, + requests=Resources(cpu="4", mem="24Gi", gpu="1"), + limits=Resources(cpu="4", mem="24Gi", gpu="1"), + pod_template=_large_shm_pod_template(), + environment={"MLFLOW_TRACKING_URI": MLFLOW_URI}, +) +def train_reactive_multitask_stage( + shards: List[FlyteDirectory], + dataset: Dataset, + stage: str, + parent_checkpoint: Optional[FlyteFile] = None, + backbone: Backbone = Backbone.SWIN_V2_TINY, + epochs: int = 3, + batch_size: int = 2, + lr: float = 1e-4, + weight_decay: float = 1e-2, + grad_clip: float = 1.0, + val_fraction: float = 0.1, + num_workers: int = 0, + training_seed: int = 149, + bev_weight: float = 1.0, + route_weight: float = 1.0, + bev_pos_weights: Optional[List[float]] = None, + corridor_pos_weight: float = 1.0, +) -> TrainOutput: + """Train one locked Reactive stage on already packed immutable shards.""" + import hashlib + import json + import os + import random + from pathlib import Path + + import mlflow + import numpy as np + import torch + from flytekit import current_context + + from data_parsing.pre_extracted import make_multi_dataset_loader + from model_components.auto_e2e import AutoE2E + from navigation.geometry import AUTOE2E_NAVIGATION_GEOMETRY + from Platform.pipelines.training_checkpoint import stable_digest + from training.reactive_multitask import ( + SIMPLE_XY_IMITATION_OBJECTIVE_VERSION, + ReactiveMultitaskObjective, + ReactiveTrainingStage, + configure_model_for_stage, + reactive_model_kwargs, + ) + from training.reactive_stage_runner import ( + evaluate_reactive_xy, + inspect_reactive_checkpoint_identity, + load_stage_a_parent, + run_reactive_epoch, + save_reactive_checkpoint, + ) + + try: + training_stage = ReactiveTrainingStage(stage) + except ValueError as error: + raise ValueError(f"unsupported Reactive training stage {stage!r}") from error + expected_dataset = ( + Dataset.NUPLAN + if training_stage is ReactiveTrainingStage.NUPLAN_FULL + else Dataset.L2D + ) + if dataset is not expected_dataset: + raise ValueError( + f"{training_stage.value} requires dataset={expected_dataset.value}" + ) + if ( + training_stage is ReactiveTrainingStage.NUPLAN_FULL + and parent_checkpoint is not None + ): + raise ValueError("Stage A must not load a parent checkpoint") + if ( + training_stage is ReactiveTrainingStage.L2D_CONTINUATION + and parent_checkpoint is None + ): + raise ValueError("Stage B requires the exact Stage A checkpoint") + if epochs <= 0 or batch_size <= 0: + raise ValueError("epochs and batch_size must be positive") + if lr <= 0.0 or weight_decay < 0.0 or grad_clip <= 0.0: + raise ValueError("optimizer parameters are invalid") + if not 0.0 < val_fraction < 1.0: + raise ValueError("val_fraction must be between zero and one") + if num_workers < 0: + raise ValueError("num_workers must be non-negative") + normalized_bev_pos_weights = ( + [1.0] * 8 + if bev_pos_weights is None + else bev_pos_weights + ) + if len(normalized_bev_pos_weights) != 8 or any( + not np.isfinite(value) or value <= 0.0 + for value in normalized_bev_pos_weights + ): + raise ValueError("bev_pos_weights must contain eight positive values") + if not 0 <= training_seed <= 2**32 - 1: + raise ValueError("training_seed is outside uint32") + + random.seed(training_seed) + np.random.seed(training_seed) + torch.manual_seed(training_seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(training_seed) + torch.backends.cudnn.benchmark = False + torch.backends.cudnn.deterministic = True + if num_workers: + torch.multiprocessing.set_sharing_strategy("file_system") + + shard_dirs: list[str] = [] + manifest_identities: list[dict] = [] + view_counts: set[int] = set() + expected_geometry = AUTOE2E_NAVIGATION_GEOMETRY.contract() + for shard in shards: + shard_uri = str( + getattr(shard, "remote_source", "") or shard + ) + shard_dir = _loader_download_dir(shard) + manifest_path = Path(shard_dir) / "manifest.json" + if not manifest_path.is_file(): + raise FileNotFoundError( + f"packed shard manifest is missing: {manifest_path}" + ) + manifest_bytes = manifest_path.read_bytes() + try: + manifest = json.loads(manifest_bytes) + except json.JSONDecodeError as error: + raise ValueError( + f"packed shard manifest is invalid: {manifest_path}" + ) from error + if manifest.get("dataset") != dataset.value: + continue + sample_count = int(manifest.get("total_samples", 0)) + if sample_count <= 0: + continue + required_flags = { + "has_reactive_navigation": True, + "has_route_reconstruction": True, + "has_trajectory_xy": True, + } + if training_stage is ReactiveTrainingStage.NUPLAN_FULL: + required_flags["has_bev_segmentation"] = True + mismatched_flags = { + key: manifest.get(key) + for key, expected in required_flags.items() + if manifest.get(key) is not expected + } + if mismatched_flags: + raise ValueError( + "packed Reactive target coverage is incomplete: " + f"{mismatched_flags} ({manifest_path})" + ) + if manifest.get("navigation_geometry") != expected_geometry: + raise ValueError( + "packed navigation geometry differs from the common " + f"450x300 contract: {manifest_path}" + ) + if int(manifest.get("map_context_channels", 0)) != 14: + raise ValueError("Reactive stages require 14 map channels") + if int(manifest.get("route_channels", 0)) != 2: + raise ValueError("Reactive stages require two route channels") + num_views = int(manifest.get("num_views", 0)) + if num_views <= 0: + raise ValueError("Reactive stage shard has no camera views") + view_counts.add(num_views) + shard_dirs.append(shard_dir) + manifest_identities.append({ + "dataset": dataset.value, + "manifest_sha256": hashlib.sha256( + manifest_bytes + ).hexdigest(), + "partition_id": manifest.get("partition_id"), + "shard_names": list(manifest.get("shard_names", [])), + "source_revision": manifest.get("source_revision"), + "total_samples": sample_count, + "uri": shard_uri, + }) + if not shard_dirs: + raise ValueError( + f"no non-empty packed shards matched {dataset.value}" + ) + if len(view_counts) != 1: + raise ValueError( + f"Reactive stage mixes camera counts: {sorted(view_counts)}" + ) + manifest_identities.sort( + key=lambda item: ( + str(item["partition_id"]), + str(item["shard_names"]), + str(item["uri"]), + ) + ) + dataset_manifest_sha256 = stable_digest(manifest_identities) + num_views = next(iter(view_counts)) + + device = torch.device( + "cuda" if torch.cuda.is_available() else "cpu" + ) + constructor_kwargs = reactive_model_kwargs( + training_stage, + num_views=num_views, + ) + model = AutoE2E( + backbone=backbone.value, + embed_dim=256, + is_pretrained=( + training_stage is ReactiveTrainingStage.NUPLAN_FULL + ), + **constructor_kwargs, + ).to(device) + lineage: dict[str, str] = {} + if parent_checkpoint is not None: + lineage.update( + load_stage_a_parent( + model, + str(parent_checkpoint.download()), + ) + ) + configure_model_for_stage(model, training_stage) + objective = ReactiveMultitaskObjective( + training_stage, + bev_pos_weight=normalized_bev_pos_weights, + bev_weight=bev_weight, + route_weight=route_weight, + corridor_pos_weight=corridor_pos_weight, + ).to(device) + trainable = [ + parameter + for parameter in model.parameters() + if parameter.requires_grad + ] + optimizer = torch.optim.AdamW( + trainable, + lr=lr, + weight_decay=weight_decay, + ) + scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau( + optimizer, + mode="min", + factor=0.5, + patience=1, + threshold=1e-4, + threshold_mode="abs", + ) + train_loader = make_multi_dataset_loader( + shard_dirs, + batch_size=batch_size, + num_workers=num_workers, + split="train", + val_fraction=val_fraction, + shuffle=1000, + shuffle_seed=training_seed, + pin_memory=(device.type == "cuda"), + decode_future_frames=False, + ) + validation_loader = make_multi_dataset_loader( + shard_dirs, + batch_size=batch_size, + num_workers=min(num_workers, 1), + split="val", + val_fraction=val_fraction, + shuffle=0, + pin_memory=(device.type == "cuda"), + max_active_loaders=1, + decode_future_frames=False, + ) + + output_dir = Path("/tmp/reactive-multistage") / training_stage.value + output_dir.mkdir(parents=True, exist_ok=True) + checkpoint_path = output_dir / "best.pt" + metadata_path = output_dir / "metadata.json" + history = [] + best_ade = float("inf") + best_epoch = 0 + best_sha256 = "" + model_config = { + "backbone": backbone.value, + "embed_dim": 256, + # Evaluation must never download initialization weights. + "is_pretrained": False, + **constructor_kwargs, + } + + mlflow.set_tracking_uri(os.environ["MLFLOW_TRACKING_URI"]) + mlflow.set_experiment("reactive-multistage") + ctx = current_context() + with mlflow.start_run() as active_run: + run_id = active_run.info.run_id + mlflow.log_params({ + "training_stage": training_stage.value, + "dataset": dataset.value, + "training_objective_version": ( + SIMPLE_XY_IMITATION_OBJECTIVE_VERSION + ), + "navigation_geometry_id": ( + AUTOE2E_NAVIGATION_GEOMETRY.geometry_id + ), + "planner_mode": "gru", + "enable_world_model": False, + "enable_reasoning": False, + "epochs": epochs, + "batch_size": batch_size, + "lr": lr, + "bev_weight": bev_weight, + "route_weight": route_weight, + }) + for epoch in range(1, epochs + 1): + train_metrics = run_reactive_epoch( + model, + train_loader, + objective, + optimizer, + device=device, + grad_clip=grad_clip, + ) + validation_metrics = evaluate_reactive_xy( + model, + validation_loader, + device=device, + ) + scheduler.step(validation_metrics["ade_6p4s_m"]) + record = { + "epoch": epoch, + "train": train_metrics, + "validation": validation_metrics, + "lr": float(optimizer.param_groups[0]["lr"]), + } + history.append(record) + mlflow.log_metrics( + { + **{ + f"train/{name}": value + for name, value in train_metrics.items() + }, + **{ + f"val/{name}": value + for name, value in validation_metrics.items() + }, + }, + step=epoch, + ) + if validation_metrics["ade_6p4s_m"] < best_ade: + best_ade = validation_metrics["ade_6p4s_m"] + best_epoch = epoch + best_sha256 = save_reactive_checkpoint( + checkpoint_path, + model, + stage=training_stage, + dataset_manifest_sha256=dataset_manifest_sha256, + epoch=epoch, + model_config=model_config, + optimizer=optimizer, + scheduler=scheduler, + metrics=validation_metrics, + training_state={ + "run_id": run_id, + "flyte_execution_id": ( + ctx.execution_id.name + if ctx.execution_id + else "local" + ), + }, + lineage=lineage, + ) + mlflow.log_artifact(str(checkpoint_path), artifact_path="checkpoints") + + checkpoint_identity = inspect_reactive_checkpoint_identity( + checkpoint_path + ) + metadata = { + "schema_version": "reactive_multistage_training_v1", + "training_stage": training_stage.value, + "dataset": dataset.value, + "dataset_manifest_sha256": dataset_manifest_sha256, + "best_epoch": best_epoch, + "best_checkpoint_sha256": best_sha256, + "best_checkpoint_identity": checkpoint_identity, + "history": history, + "lineage": lineage, + "model_config": model_config, + "objective": { + "version": SIMPLE_XY_IMITATION_OBJECTIVE_VERSION, + "bev_weight": ( + bev_weight + if training_stage is ReactiveTrainingStage.NUPLAN_FULL + else 0.0 + ), + "route_weight": route_weight, + }, + } + metadata_path.write_text( + json.dumps( + metadata, + allow_nan=False, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="ascii", + ) + return TrainOutput( + checkpoint=FlyteFile(str(checkpoint_path)), + metadata=FlyteFile(str(metadata_path)), + ) + + +@task( + container_image=EVAL_IMAGE, + requests=Resources(cpu="4", mem="24Gi", gpu="1"), + limits=Resources(cpu="4", mem="24Gi", gpu="1"), + pod_template=_large_shm_pod_template(), +) +def evaluate_reactive_transfer_matrix( + stage_a_checkpoint: FlyteFile, + stage_b_checkpoint: FlyteFile, + nuplan_shards: List[FlyteDirectory], + l2d_shards: List[FlyteDirectory], + batch_size: int = 2, + val_fraction: float = 0.1, + num_workers: int = 0, +) -> ReactiveRetentionOutput: + """Evaluate Stage A/B on one frozen nuPlan/L2D validation split.""" + import hashlib + import json + import tempfile + from pathlib import Path + + import torch + + from data_parsing.pre_extracted import ( + discover_split_inventory, + make_multi_dataset_loader, + ) + from data_processing.dataset_snapshot import split_bucket + from navigation.geometry import AUTOE2E_NAVIGATION_GEOMETRY + from Platform.pipelines.inference import load_policy + from Platform.pipelines.training_checkpoint import stable_digest + from training.reactive_multitask import ReactiveTrainingStage + from training.reactive_stage_runner import ( + evaluate_reactive_multitask, + inspect_reactive_checkpoint_identity, + ) + + if batch_size <= 0 or num_workers < 0: + raise ValueError("invalid retention evaluation loader settings") + if not 0.0 < val_fraction < 1.0: + raise ValueError("val_fraction must be between zero and one") + + expected_geometry = AUTOE2E_NAVIGATION_GEOMETRY.contract() + + def resolve_dataset( + shards: List[FlyteDirectory], + dataset: Dataset, + ) -> tuple[list[str], str, dict]: + directories: list[str] = [] + identities: list[dict] = [] + for shard in shards: + directory = _loader_download_dir(shard) + manifest_path = Path(directory) / "manifest.json" + if not manifest_path.is_file(): + raise FileNotFoundError( + f"packed shard manifest is missing: {manifest_path}" + ) + payload = manifest_path.read_bytes() + manifest = json.loads(payload) + if manifest.get("dataset") != dataset.value: + continue + if int(manifest.get("total_samples", 0)) <= 0: + continue + if manifest.get("navigation_geometry") != expected_geometry: + raise ValueError( + "retention dataset navigation geometry differs from " + "the common contract" + ) + required = { + "has_reactive_navigation": True, + "has_route_reconstruction": True, + "has_trajectory_xy": True, + } + if dataset is Dataset.NUPLAN: + required["has_bev_segmentation"] = True + mismatches = { + key: manifest.get(key) + for key, expected in required.items() + if manifest.get(key) is not expected + } + if mismatches: + raise ValueError( + "retention dataset target coverage is incomplete: " + f"{mismatches}" + ) + directories.append(directory) + identities.append({ + "dataset": dataset.value, + "manifest_sha256": hashlib.sha256(payload).hexdigest(), + "partition_id": manifest.get("partition_id"), + "shard_names": list(manifest.get("shard_names", [])), + "source_revision": manifest.get("source_revision"), + "total_samples": int(manifest["total_samples"]), + "uri": str( + getattr(shard, "remote_source", "") or shard + ), + }) + if not directories: + raise ValueError( + f"no non-empty retention shards matched {dataset.value}" + ) + identities.sort( + key=lambda item: ( + str(item["partition_id"]), + str(item["shard_names"]), + str(item["uri"]), + ) + ) + inventory = discover_split_inventory(directories) + buckets = 10 + validation_bucket_count = max( + 1, + min(buckets - 1, round(val_fraction * buckets)), + ) + validation_groups = tuple( + group_uid + for group_uid in inventory.group_uids + if split_bucket(group_uid, buckets) < validation_bucket_count + ) + if not validation_groups: + raise ValueError( + f"{dataset.value} has no groups in the frozen validation split" + ) + expected_count, expected_uid_digest = ( + inventory.sample_identity_for_groups(validation_groups) + ) + return directories, stable_digest(identities), { + "dataset": dataset.value, + "manifest_digest": stable_digest(identities), + "validation_group_count": len(validation_groups), + "validation_group_sha256": hashlib.sha256( + "\n".join(validation_groups).encode("utf-8") + ).hexdigest(), + "validation_groups": list(validation_groups), + "expected_sample_count": expected_count, + "expected_sample_uid_sha256": expected_uid_digest, + } + + nuplan_directories, nuplan_digest, nuplan_split = resolve_dataset( + nuplan_shards, + Dataset.NUPLAN, + ) + l2d_directories, l2d_digest, l2d_split = resolve_dataset( + l2d_shards, + Dataset.L2D, + ) + dataset_specs = { + "nuplan": ( + nuplan_directories, + nuplan_split, + ), + "l2d": ( + l2d_directories, + l2d_split, + ), + } + + stage_a_path = str(stage_a_checkpoint.download()) + stage_b_path = str(stage_b_checkpoint.download()) + device = torch.device( + "cuda" if torch.cuda.is_available() else "cpu" + ) + if torch.backends.cudnn.is_available(): + torch.backends.cudnn.benchmark = False + torch.backends.cudnn.deterministic = True + + stage_a_identity = inspect_reactive_checkpoint_identity(stage_a_path) + stage_b_identity = inspect_reactive_checkpoint_identity(stage_b_path) + stage_a_sha256 = stage_a_identity["checkpoint_sha256"] + stage_b_sha256 = stage_b_identity["checkpoint_sha256"] + + loader_factories = { + dataset_name: functools.partial( + make_multi_dataset_loader, + directories, + batch_size=batch_size, + num_workers=num_workers, + split="val", + val_fraction=0.0, + shuffle=0, + pin_memory=(device.type == "cuda"), + max_active_loaders=1, + validation_group_uids=( + split_metadata["validation_groups"] + ), + decode_future_frames=False, + ) + for dataset_name, ( + directories, + split_metadata, + ) in dataset_specs.items() + } + matrix: dict[str, dict[str, dict]] = { + "stage_a": {}, + "stage_b": {}, + } + checkpoint_specs = ( + ( + "stage_a", + stage_a_path, + ReactiveTrainingStage.NUPLAN_FULL.value, + nuplan_digest, + ), + ( + "stage_b", + stage_b_path, + ReactiveTrainingStage.L2D_CONTINUATION.value, + l2d_digest, + ), + ) + checkpoint_configs = {} + for ( + checkpoint_name, + checkpoint_path, + expected_stage, + expected_manifest_digest, + ) in checkpoint_specs: + model, config, loaded_sha256 = load_policy( + checkpoint_path, + device, + ) + expected_sha256 = ( + stage_a_sha256 + if checkpoint_name == "stage_a" + else stage_b_sha256 + ) + if loaded_sha256 != expected_sha256: + raise ValueError( + f"{checkpoint_name} identity changed while loading" + ) + if config.get("training_stage") != expected_stage: + raise ValueError( + f"{checkpoint_name} checkpoint has the wrong training stage" + ) + if config.get( + "dataset_manifest_sha256" + ) != expected_manifest_digest: + raise ValueError( + f"{checkpoint_name} checkpoint was trained on different shards" + ) + if ( + checkpoint_name == "stage_b" + and config.get("stage_a_parent_checkpoint_sha256") + != stage_a_sha256 + ): + raise ValueError( + "Stage B lineage does not reference the supplied " + "Stage A checkpoint" + ) + checkpoint_configs[checkpoint_name] = config + for dataset_name, loader_factory in loader_factories.items(): + matrix[checkpoint_name][dataset_name] = ( + evaluate_reactive_multitask( + model, + loader_factory(), + device=device, + ) + ) + del model + if device.type == "cuda": + torch.cuda.empty_cache() + + stage_b_config = checkpoint_configs["stage_b"] + for dataset_name in dataset_specs: + stage_a_metrics = matrix["stage_a"][dataset_name] + stage_b_metrics = matrix["stage_b"][dataset_name] + if ( + stage_a_metrics["sample_count"] + != stage_b_metrics["sample_count"] + or stage_a_metrics["sample_uid_sha256"] + != stage_b_metrics["sample_uid_sha256"] + ): + raise ValueError( + "Stage A and Stage B retention cells used different " + f"{dataset_name} validation samples" + ) + for checkpoint_name in ("stage_a", "stage_b"): + for dataset_name, ( + _, + split_metadata, + ) in dataset_specs.items(): + metrics = matrix[checkpoint_name][dataset_name] + if metrics["sample_count"] != ( + split_metadata["expected_sample_count"] + ): + raise ValueError( + "retention evaluation sample count differs from " + f"the frozen inventory for {dataset_name}" + ) + if metrics["sample_uid_sha256"] != ( + split_metadata["expected_sample_uid_sha256"] + ): + raise ValueError( + "retention evaluation sample UID digest differs from " + f"the frozen inventory for {dataset_name}" + ) + + report = { + "schema_version": "reactive_transfer_matrix_v1", + "checkpoint_lineage": { + "stage_a_checkpoint_sha256": stage_a_sha256, + "stage_b_checkpoint_sha256": stage_b_sha256, + "stage_b_parent_checkpoint_sha256": stage_b_config[ + "stage_a_parent_checkpoint_sha256" + ], + "stage_a_config_digest": stage_a_identity["config_sha256"], + "stage_b_config_digest": stage_b_identity["config_sha256"], + "stage_a_model_state_sha256": ( + stage_a_identity["model_state_sha256"] + ), + "stage_b_model_state_sha256": ( + stage_b_identity["model_state_sha256"] + ), + }, + "datasets": { + "nuplan": nuplan_split, + "l2d": l2d_split, + }, + "matrix": matrix, + } + report_payload = ( + json.dumps( + report, + allow_nan=False, + indent=2, + sort_keys=True, + ) + + "\n" + ).encode("ascii") + report_sha256 = hashlib.sha256(report_payload).hexdigest() + output_path = ( + Path(tempfile.mkdtemp(prefix="reactive-retention-")) + / "retention-report.json" + ) + output_path.write_bytes(report_payload) + return ReactiveRetentionOutput( + report=FlyteFile(str(output_path)), + report_sha256=report_sha256, + ) + + +@task( + container_image=EVAL_IMAGE, + requests=Resources(cpu="4", mem="24Gi", gpu="1"), + limits=Resources(cpu="4", mem="24Gi", gpu="1"), + pod_template=_large_shm_pod_template(), +) +def precompute_semantic_occupancy_artifacts( + checkpoint: FlyteFile, + shard_dirs: List[FlyteDirectory], + dataset: str, + dataset_version: str, + dataset_manifest_sha256: str, + artifacts_bucket: str, + publication_timestamp: str, + repository_revision: str, + aws_region: str = "us-west-2", + batch_size: int = 2, + num_workers: int = 0, +) -> SemanticOccupancyPrecomputeOutput: + """Precompute immutable 2D semantic occupancy bodies per packed tar.""" + import hashlib + import json + import os + import re + from pathlib import Path + + import boto3 + import torch + + from data_parsing.pre_extracted import make_pre_extracted_loader + from Platform.pipelines.inference import load_policy + from Platform.pipelines.overlay_tasks import _put_s3_immutable + from Platform.pipelines.occupancy_store import ( + encode_occupancy_set_manifest, + occupancy_model_artifact_id, + occupancy_set_manifest, + occupancy_set_s3_key, + ) + from Platform.pipelines.semantic_occupancy import ( + SEMANTIC_OCCUPANCY_CLASS_NAMES, + SEMANTIC_OCCUPANCY_GEOMETRY_ID, + SEMANTIC_OCCUPANCY_HEAD_VERSION, + SEMANTIC_OCCUPANCY_SCHEMA, + SEMANTIC_OCCUPANCY_TAXONOMY_VERSION, + encode_semantic_occupancy, + infer_semantic_occupancy, + semantic_occupancy_s3_key, + ) + + if not re.fullmatch(r"[0-9a-f]{64}", dataset_manifest_sha256): + raise ValueError( + "dataset_manifest_sha256 must be a lowercase SHA-256" + ) + for name, value in ( + ("dataset", dataset), + ("dataset_version", dataset_version), + ("artifacts_bucket", artifacts_bucket), + ("publication_timestamp", publication_timestamp), + ("repository_revision", repository_revision), + ("aws_region", aws_region), + ): + if not value: + raise ValueError(f"{name} must not be empty") + if "/" in dataset or "\\" in dataset: + raise ValueError("dataset must be one path segment") + if not shard_dirs: + raise ValueError("shard_dirs must not be empty") + if batch_size <= 0 or num_workers < 0: + raise ValueError("invalid semantic occupancy loader settings") + + packed_shards = [] + seen_shards = set() + for shard_directory in shard_dirs: + local_directory = Path(shard_directory.download()) + if not (local_directory / "manifest.json").is_file(): + raise FileNotFoundError( + f"packed manifest missing: {local_directory}" + ) + for tar_path in sorted(local_directory.glob("*.tar")): + if tar_path.name in seen_shards: + raise ValueError( + "semantic occupancy shard names are not unique: " + f"{tar_path.name}" + ) + seen_shards.add(tar_path.name) + packed_shards.append((local_directory, tar_path)) + if not packed_shards: + raise ValueError("packed directories contain no tar shards") + packed_shards.sort(key=lambda item: item[1].name) + + os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8" + torch.use_deterministic_algorithms(True) + if torch.backends.cudnn.is_available(): + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + torch.manual_seed(0) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(0) + if num_workers: + torch.multiprocessing.set_sharing_strategy("file_system") + device = torch.device( + "cuda" if torch.cuda.is_available() else "cpu" + ) + checkpoint_path = str(checkpoint.download()) + model, config, checkpoint_sha256 = load_policy( + checkpoint_path, + device, + ) + config_payload = json.dumps( + config, + allow_nan=False, + separators=(",", ":"), + sort_keys=True, + ).encode("ascii") + model_source = { + "code_license_spdx": "Apache-2.0", + "config": ( + "embedded-checkpoint-config-sha256:" + f"{hashlib.sha256(config_payload).hexdigest()}" + ), + "license_spdx": "NOASSERTION", + "repository": ( + "https://github.com/autowarefoundation/auto_e2e" + ), + "repository_revision": repository_revision, + "training_data_license_spdx": "NOASSERTION", + "weight_sha256": checkpoint_sha256, + "weight_source_url": f"urn:sha256:{checkpoint_sha256}", + } + producer_config = { + "batch_size": batch_size, + "cublas_workspace_config": os.environ["CUBLAS_WORKSPACE_CONFIG"], + "deterministic_algorithms": True, + "num_workers": num_workers, + "probability_encoding": "uint8-rint-gzip-level-6-v1", + "random_seed": 0, + } + model_artifact_id = occupancy_model_artifact_id( + artifact_kind="native-semantic-occupancy", + artifact_schema=SEMANTIC_OCCUPANCY_SCHEMA, + geometry_id=SEMANTIC_OCCUPANCY_GEOMETRY_ID, + head_version=SEMANTIC_OCCUPANCY_HEAD_VERSION, + input_contract="autoe2e-packed-calibrated-camera-v1", + model_source=model_source, + producer_config=producer_config, + taxonomy_version=SEMANTIC_OCCUPANCY_TAXONOMY_VERSION, + ) + manifest_key = occupancy_set_s3_key( + dataset, + dataset_version, + model_artifact_id, + dataset_manifest_sha256, + ) + if not config.get("enable_bev_segmentation", False): + raise ValueError("checkpoint has no BEV segmentation head") + + s3 = boto3.client("s3", region_name=aws_region) + entries = [] + total_samples = 0 + for local_directory, tar_path in packed_shards: + loader = make_pre_extracted_loader( + str(local_directory), + batch_size=batch_size, + num_workers=num_workers, + split="all", + val_fraction=0.0, + shuffle=0, + pin_memory=(device.type == "cuda"), + prefetch_factor=1, + shard_files=[tar_path], + decode_history_frames=False, + decode_future_frames=False, + ) + ( + sample_uids, + probability, + teacher, + valid_mask, + ) = infer_semantic_occupancy( + model, + loader, + device=device, + ) + payload = encode_semantic_occupancy( + sample_uids, + probability, + teacher=teacher, + valid_mask=valid_mask, + ) + payload_sha256 = hashlib.sha256(payload).hexdigest() + key = semantic_occupancy_s3_key( + model_artifact_id, + dataset_manifest_sha256, + dataset, + tar_path.name, + ) + _put_s3_immutable( + s3, + bucket=artifacts_bucket, + key=key, + payload=payload, + metadata={ + "checkpoint-sha256": checkpoint_sha256, + "dataset-manifest-sha256": ( + dataset_manifest_sha256 + ), + "geometry-id": SEMANTIC_OCCUPANCY_GEOMETRY_ID, + "head-version": SEMANTIC_OCCUPANCY_HEAD_VERSION, + "model-artifact-id": model_artifact_id, + "payload-sha256": payload_sha256, + "sample-count": str(len(sample_uids)), + "schema": SEMANTIC_OCCUPANCY_SCHEMA, + "taxonomy-version": ( + SEMANTIC_OCCUPANCY_TAXONOMY_VERSION + ), + }, + content_type=( + "application/vnd.auto-e2e.semantic-occupancy" + ), + content_encoding="gzip", + ) + entries.append({ + "byte_size": len(payload), + "sample_count": len(sample_uids), + "s3_key": key, + "sha256": payload_sha256, + "shard": tar_path.name, + "teacher_present": teacher is not None, + }) + total_samples += len(sample_uids) + entries.sort(key=lambda entry: entry["shard"]) + teacher_available = all( + entry["teacher_present"] + for entry in entries + ) + limitations = [ + ( + "Predictions use the checkpoint's native BEV segmentation head " + "without viewer-side geometry correction." + ), + ] + if not teacher_available: + limitations.append( + "Teacher and Error views are unavailable because KITScenes " + "packed shards do not contain perception ground truth." + ) + manifest = occupancy_set_manifest( + artifact_kind="native-semantic-occupancy", + artifact_schema=SEMANTIC_OCCUPANCY_SCHEMA, + created_at=publication_timestamp, + dataset=dataset, + dataset_version=dataset_version, + dataset_manifest_sha256=dataset_manifest_sha256, + display_name="AutoE2E Reactive BEV segmentation", + geometry_id=SEMANTIC_OCCUPANCY_GEOMETRY_ID, + head_version=SEMANTIC_OCCUPANCY_HEAD_VERSION, + input_contract="autoe2e-packed-calibrated-camera-v1", + limitations=limitations, + model_artifact_id=model_artifact_id, + model_family="AutoE2E Reactive", + model_source=model_source, + producer_config=producer_config, + shards=entries, + supported_classes=SEMANTIC_OCCUPANCY_CLASS_NAMES, + taxonomy_version=SEMANTIC_OCCUPANCY_TAXONOMY_VERSION, + teacher_available=teacher_available, + ) + manifest_payload, manifest_sha256 = encode_occupancy_set_manifest( + manifest + ) + _put_s3_immutable( + s3, + bucket=artifacts_bucket, + key=manifest_key, + payload=manifest_payload, + metadata={ + "checkpoint-sha256": checkpoint_sha256, + "dataset-manifest-sha256": dataset_manifest_sha256, + "manifest-sha256": manifest_sha256, + "model-artifact-id": model_artifact_id, + "sample-count": str(total_samples), + "schema": manifest["schema_version"], + }, + content_type="application/json", + ) + return SemanticOccupancyPrecomputeOutput( + manifest_key=manifest_key, + manifest_sha256=manifest_sha256, + checkpoint_sha256=checkpoint_sha256, + shard_count=len(entries), + sample_count=total_samples, + ) + + +@task( + container_image=BEVFORMER_V2_IMAGE, + requests=Resources(cpu="4", mem="28Gi", gpu="1"), + limits=Resources(cpu="4", mem="28Gi", gpu="1"), + pod_template=_large_shm_pod_template(), +) +def precompute_bevformer_v2_occupancy_artifacts( + checkpoint: FlyteFile, + shard_dirs: List[FlyteDirectory], + dataset: str, + dataset_version: str, + dataset_manifest_sha256: str, + artifacts_bucket: str, + publication_timestamp: str, + aws_region: str = "us-west-2", + repository_path: str = "/opt/BEVFormer", + score_threshold: float = 0.2, +) -> SemanticOccupancyPrecomputeOutput: + """Publish detection-derived KITScenes occupancy from official V2 weights.""" + import hashlib + import json + import os + import re + from pathlib import Path + + import boto3 + import numpy as np + import torch + + from Platform.pipelines.bevformer_v2_occupancy import ( + BEVFORMER_V2_ARTIFACT_KIND, + BEVFORMER_V2_CODE_LICENSE_SPDX, + BEVFORMER_V2_CONFIG_NAME, + BEVFORMER_V2_FRAMES, + BEVFORMER_V2_HEAD_VERSION, + BEVFORMER_V2_REPOSITORY, + BEVFORMER_V2_REVISION, + BEVFORMER_V2_SUPPORTED_SEMANTIC_CLASSES, + BEVFORMER_V2_TRAINING_DATA_LICENSE_SPDX, + BEVFORMER_V2_WEIGHT_LICENSE_SPDX, + BEVFORMER_V2_WEIGHT_SHA256, + BEVFORMER_V2_WEIGHT_SOURCE_URL, + provenance, + ) + from Platform.pipelines.bevformer_v2_runtime import ( + BEVFORMER_V2_IMAGE_HEIGHT, + BEVFORMER_V2_IMAGE_WIDTH, + infer_bevformer_frame, + iter_packed_bevformer_frames, + load_official_bevformer_v2, + remember_history_frame, + temporal_frames_for, + ) + from Platform.pipelines.occupancy_store import ( + encode_occupancy_set_manifest, + occupancy_model_artifact_id, + occupancy_set_manifest, + occupancy_set_s3_key, + ) + from Platform.pipelines.overlay_tasks import _put_s3_immutable + from Platform.pipelines.semantic_occupancy import ( + SEMANTIC_OCCUPANCY_GEOMETRY_ID, + SEMANTIC_OCCUPANCY_SCHEMA, + SEMANTIC_OCCUPANCY_TAXONOMY_VERSION, + encode_semantic_occupancy, + quantize_semantic_occupancy, + semantic_occupancy_s3_key, + ) + + if dataset != "kitscenes": + raise ValueError("BEVFormer V2 occupancy supports only KITScenes") + if not re.fullmatch(r"v[1-9][0-9]*\.[0-9]+", dataset_version): + raise ValueError("dataset_version must match v.") + if not re.fullmatch(r"[0-9a-f]{64}", dataset_manifest_sha256): + raise ValueError( + "dataset_manifest_sha256 must be a lowercase SHA-256" + ) + for name, value in ( + ("artifacts_bucket", artifacts_bucket), + ("publication_timestamp", publication_timestamp), + ("aws_region", aws_region), + ("repository_path", repository_path), + ): + if not value: + raise ValueError(f"{name} must not be empty") + if not shard_dirs: + raise ValueError("shard_dirs must not be empty") + if not 0.0 <= score_threshold <= 1.0: + raise ValueError("score_threshold must be in [0,1]") + + packed_shards = [] + seen_shards = set() + for shard_directory in shard_dirs: + local_directory = Path(shard_directory.download()) + packed_manifest_path = local_directory / "manifest.json" + if not packed_manifest_path.is_file(): + raise FileNotFoundError( + f"packed manifest missing: {local_directory}" + ) + packed_manifest = json.loads( + packed_manifest_path.read_text(encoding="utf-8") + ) + packed_dataset_version = packed_manifest.get( + "dataset_version", + packed_manifest.get("version"), + ) + if ( + packed_manifest.get("dataset") != dataset + or packed_dataset_version != dataset_version + or int(packed_manifest.get("num_views", 0)) + != 6 + or not packed_manifest.get("has_gps", False) + ): + raise ValueError( + "packed KITScenes manifest differs from the BEVFormer input " + f"contract: {packed_manifest_path}" + ) + for tar_path in sorted(local_directory.glob("*.tar")): + if tar_path.name in seen_shards: + raise ValueError( + "semantic occupancy shard names are not unique: " + f"{tar_path.name}" + ) + seen_shards.add(tar_path.name) + packed_shards.append(tar_path) + if not packed_shards: + raise ValueError("packed directories contain no tar shards") + packed_shards.sort(key=lambda path: path.name) + + model_source = { + "code_license_spdx": BEVFORMER_V2_CODE_LICENSE_SPDX, + "config": BEVFORMER_V2_CONFIG_NAME, + "license_spdx": BEVFORMER_V2_WEIGHT_LICENSE_SPDX, + "repository": BEVFORMER_V2_REPOSITORY, + "repository_revision": BEVFORMER_V2_REVISION, + "training_data_license_spdx": ( + BEVFORMER_V2_TRAINING_DATA_LICENSE_SPDX + ), + "weight_sha256": BEVFORMER_V2_WEIGHT_SHA256, + "weight_source_url": BEVFORMER_V2_WEIGHT_SOURCE_URL, + } + producer_config = { + "cublas_workspace_config": ":4096:8", + "deterministic_algorithms": True, + "image_height": BEVFORMER_V2_IMAGE_HEIGHT, + "image_width": BEVFORMER_V2_IMAGE_WIDTH, + "max_detections": 300, + "probability_encoding": "uint8-rint-gzip-level-6-v1", + "random_seed": 0, + "score_threshold": score_threshold, + "temporal_frame_offsets": list(BEVFORMER_V2_FRAMES), + } + model_artifact_id = occupancy_model_artifact_id( + artifact_kind=BEVFORMER_V2_ARTIFACT_KIND, + artifact_schema=SEMANTIC_OCCUPANCY_SCHEMA, + geometry_id=SEMANTIC_OCCUPANCY_GEOMETRY_ID, + head_version=BEVFORMER_V2_HEAD_VERSION, + input_contract=( + "kitscenes-packed-256-square-six-camera-to-" + "bevformer-640x256-v1" + ), + model_source=model_source, + producer_config=producer_config, + taxonomy_version=SEMANTIC_OCCUPANCY_TAXONOMY_VERSION, + ) + manifest_key = occupancy_set_s3_key( + dataset, + dataset_version, + model_artifact_id, + dataset_manifest_sha256, + ) + + os.environ["CUBLAS_WORKSPACE_CONFIG"] = ( + producer_config["cublas_workspace_config"] + ) + torch.use_deterministic_algorithms(True) + if torch.backends.cudnn.is_available(): + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + np.random.seed(0) + torch.manual_seed(0) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(0) + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + checkpoint_path = str(checkpoint.download()) + model, box_type_3d = load_official_bevformer_v2( + repository_path=repository_path, + checkpoint_path=checkpoint_path, + device=device, + ) + s3 = boto3.client("s3", region_name=aws_region) + entries = [] + total_samples = 0 + incomplete_history_frames = 0 + missing_history_slots = 0 + history = {} + active_episode = None + last_frame_index = None + for tar_path in packed_shards: + sample_uids = [] + probabilities = [] + for frame in iter_packed_bevformer_frames(tar_path): + if active_episode != frame.episode_id: + history.clear() + active_episode = frame.episode_id + last_frame_index = None + if ( + last_frame_index is not None + and frame.frame_index <= last_frame_index + ): + raise ValueError( + "packed KITScenes frames are not strictly increasing " + f"within episode {frame.episode_id!r}" + ) + available_history = temporal_frames_for(frame, history) + missing_slots = ( + len(BEVFORMER_V2_FRAMES) - len(available_history) + ) + if missing_slots: + incomplete_history_frames += 1 + missing_history_slots += missing_slots + probability = infer_bevformer_frame( + model, + frame, + history, + box_type_3d=box_type_3d, + device=device, + score_threshold=score_threshold, + ) + probabilities.append( + quantize_semantic_occupancy(probability) + ) + sample_uids.append(frame.sample_uid) + remember_history_frame(history, frame) + last_frame_index = frame.frame_index + if not sample_uids: + raise ValueError(f"packed shard is empty: {tar_path}") + payload = encode_semantic_occupancy( + sample_uids, + np.stack(probabilities), + ) + payload_sha256 = hashlib.sha256(payload).hexdigest() + key = semantic_occupancy_s3_key( + model_artifact_id, + dataset_manifest_sha256, + dataset, + tar_path.name, + head_version=BEVFORMER_V2_HEAD_VERSION, + ) + _put_s3_immutable( + s3, + bucket=artifacts_bucket, + key=key, + payload=payload, + metadata={ + "artifact-kind": BEVFORMER_V2_ARTIFACT_KIND, + "dataset-manifest-sha256": ( + dataset_manifest_sha256 + ), + "geometry-id": SEMANTIC_OCCUPANCY_GEOMETRY_ID, + "head-version": BEVFORMER_V2_HEAD_VERSION, + "model-artifact-id": model_artifact_id, + "payload-sha256": payload_sha256, + "sample-count": str(len(sample_uids)), + "schema": SEMANTIC_OCCUPANCY_SCHEMA, + "taxonomy-version": ( + SEMANTIC_OCCUPANCY_TAXONOMY_VERSION + ), + "weight-sha256": BEVFORMER_V2_WEIGHT_SHA256, + }, + content_type=( + "application/vnd.auto-e2e.semantic-occupancy" + ), + content_encoding="gzip", + ) + entries.append({ + "byte_size": len(payload), + "sample_count": len(sample_uids), + "s3_key": key, + "sha256": payload_sha256, + "shard": tar_path.name, + "teacher_present": False, + }) + total_samples += len(sample_uids) + metadata = provenance() + limitations = list(metadata["limitations"]) + limitations.append( + "The official weight is supplied at execution time and is not " + "redistributed in the AutoE2E container image." + ) + if incomplete_history_frames: + limitations.append( + f"{incomplete_history_frames} of {total_samples} frames had " + f"{missing_history_slots} unavailable exact t8 history slots at " + "scene or packed-sequence boundaries; unavailable inputs were " + "omitted without substituting adjacent KITScenes frames." + ) + manifest = occupancy_set_manifest( + artifact_kind=BEVFORMER_V2_ARTIFACT_KIND, + artifact_schema=SEMANTIC_OCCUPANCY_SCHEMA, + created_at=publication_timestamp, + dataset=dataset, + dataset_version=dataset_version, + dataset_manifest_sha256=dataset_manifest_sha256, + display_name="BEVFormer V2 R50 t8 detection footprints", + geometry_id=SEMANTIC_OCCUPANCY_GEOMETRY_ID, + head_version=BEVFORMER_V2_HEAD_VERSION, + input_contract=( + "kitscenes-packed-256-square-six-camera-to-" + "bevformer-640x256-v1" + ), + limitations=limitations, + model_artifact_id=model_artifact_id, + model_family="BEVFormer V2", + model_source=model_source, + producer_config=producer_config, + shards=entries, + supported_classes=( + BEVFORMER_V2_SUPPORTED_SEMANTIC_CLASSES + ), + taxonomy_version=SEMANTIC_OCCUPANCY_TAXONOMY_VERSION, + teacher_available=False, + ) + manifest_payload, manifest_sha256 = encode_occupancy_set_manifest( + manifest + ) + _put_s3_immutable( + s3, + bucket=artifacts_bucket, + key=manifest_key, + payload=manifest_payload, + metadata={ + "artifact-kind": BEVFORMER_V2_ARTIFACT_KIND, + "dataset-manifest-sha256": dataset_manifest_sha256, + "manifest-sha256": manifest_sha256, + "model-artifact-id": model_artifact_id, + "sample-count": str(total_samples), + "schema": manifest["schema_version"], + "weight-sha256": BEVFORMER_V2_WEIGHT_SHA256, + }, + content_type="application/json", + ) + return SemanticOccupancyPrecomputeOutput( + manifest_key=manifest_key, + manifest_sha256=manifest_sha256, + checkpoint_sha256=BEVFORMER_V2_WEIGHT_SHA256, + shard_count=len(entries), + sample_count=total_samples, + ) + + +# ============================================================ +# Task: Offline RL +# ============================================================ +@task( + container_image=OFFLINE_RL_IMAGE, + # requests == limits (Guaranteed QoS). + requests=Resources(cpu="4", mem="16Gi", gpu="1"), + limits=Resources(cpu="4", mem="16Gi", gpu="1"), +) +def train_offline_rl( + pretrained: FlyteFile, + shards: List[FlyteDirectory], + il_metadata: FlyteFile, + dataset: Dataset = Dataset.L2D, + epochs: int = 3, + tau: float = 0.7, + beta: float = 3.0, +) -> TrainOutput: + """Offline RL refinement of the IL checkpoint via advantage-weighted regression + against a frozen IL prior (AWR — not full IQL; no learned value network).""" + import os + import json + import torch + import numpy as np + from flytekit import current_context + + ckpt_path = pretrained.download() + il_meta = json.load(open(il_metadata.download())) + ctx = current_context() + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + print(f"Offline RL (AWR, frozen prior): epochs={epochs} beta={beta}") + + # Load IL model + from model_components.auto_e2e import AutoE2E + from data_parsing.pre_extracted import make_pre_extracted_loader + + import copy + + ckpt = torch.load( + ckpt_path, + map_location=device, + weights_only=False, + ) + config = ckpt["config"] + from training.dataset_policy import ( + adapt_egomotion_history, + training_policy_from_config, + ) + + training_policy = training_policy_from_config( + config, + dataset.value, + ) + if training_policy.validation_strategy != "hash_buckets": + raise ValueError( + "offline RL does not yet support an exact KITScenes train/holdout " + "partition; refusing to train on one shard or leak validation scenes" + ) + shard_dir = _select_shard_dir(shards, dataset) + model = AutoE2E(**_model_kwargs(config)).to(device) + model.load_state_dict(ckpt["model_state_dict"]) + + # FROZEN behavior prior = the IL checkpoint at t=0, kept fixed. The advantage + # must be measured against a policy that does NOT move with the one being + # trained; using the LIVE model for both terms makes advantage identically 0 + # (a no-op that silently reduces to plain BC). This frozen prior gives a real + # signal: "does the fine-tuned policy beat the IL prior on this sample?". + baseline_model = copy.deepcopy(model).to(device).eval() for p in baseline_model.parameters(): p.requires_grad_(False) @@ -7108,50 +9567,522 @@ def load_records(artifact, label): if primary["difference_ci95"] is not None else None ), - "primary/difference_ci95_high": ( - primary["difference_ci95"][1] - if primary["difference_ci95"] is not None - else None + "primary/difference_ci95_high": ( + primary["difference_ci95"][1] + if primary["difference_ci95"] is not None + else None + ), + "guardrail/ade_relative_regression": guardrails["ade_m"][ + "relative_regression" + ], + "guardrail/fde_relative_regression": guardrails["fde_m"][ + "relative_regression" + ], + "decision/supported": ( + 1.0 if decision["verdict"] == "supported" else 0.0 + ), + } + mlflow.log_metrics({ + key: float(value) + for key, value in metrics.items() + if value is not None + }) + mlflow.log_artifact(output_path) + + return FlyteFile(output_path) + + +@task( + container_image=EVAL_IMAGE, + requests=Resources(cpu="2", mem="8Gi", gpu="1"), + limits=Resources(cpu="2", mem="8Gi", gpu="1"), + environment={"MLFLOW_TRACKING_URI": MLFLOW_URI}, + pod_template=_large_shm_pod_template(), # /dev/shm for eval DataLoader workers (#121 P0) +) +def evaluate_rl_policy( + checkpoint: FlyteFile, + shards: List[FlyteDirectory], + train_metadata: FlyteFile, + dataset: Dataset = Dataset.L2D, +) -> EvalMetrics: + """Open-loop evaluation of the Offline-RL refined policy. + + Logs ADE/FDE, params (incl. rl/*), artifacts to the MLflow `offline-rl` + experiment and registers the refined checkpoint in the model registry. + """ + return _run_evaluation(checkpoint, shards, train_metadata, dataset, "offline-rl") + + +@task( + container_image=DATA_PREP_IMAGE, + pod_template=_data_prep_pod_template(), + requests=Resources(cpu="1", mem="2Gi", ephemeral_storage="2Gi"), + limits=Resources(cpu="1", mem="2Gi", ephemeral_storage="2Gi"), + secret_requests=[ + Secret( + group="hf-token", + key="HF_TOKEN", + mount_requirement=Secret.MountType.ENV_VAR, + ) + ], + cache=True, + cache_version="kitscenes-benchmark-inventory-v1", + retries=2, +) +def audit_kitscenes_benchmark_inventory() -> FlyteFile: + """Audit the pinned val/overlap archive inventory without downloading it.""" + import json + import os + import tempfile + + from flytekit import current_context + + from data_parsing.kit_scenes.source import ( + KITSCENES_SDK_REVISION, + fetch_archive_manifest, + resolve_inventory, + ) + + try: + token = current_context().secrets.get("hf-token", "HF_TOKEN") + except Exception: + token = os.environ.get("HF_TOKEN", "") + with tempfile.TemporaryDirectory( + prefix="kitscenes_benchmark_inventory_" + ) as tmp: + archives = fetch_archive_manifest( + tmp, + revision=KITSCENES_SOURCE_REVISION, + token=token or None, + ) + + splits = {} + total_size_bytes = 0 + total_scene_count = 0 + for source_split in ("val", "overlap_train_val"): + inventory = resolve_inventory( + archives, + split=source_split, + source_revision=KITSCENES_SOURCE_REVISION, + max_missing_scenes=0, + ) + scene_records = [ + { + "archive_path": archives[scene_id].filename, + "archive_sha256": archives[scene_id].sha256, + "archive_size_bytes": archives[scene_id].size_bytes, + "scene_id": scene_id, + } + for scene_id in inventory.selected_scene_ids + ] + split_size = sum( + int(record["archive_size_bytes"]) for record in scene_records + ) + splits[source_split] = { + **inventory.metadata(), + "archives": scene_records, + "total_size_bytes": split_size, + } + total_size_bytes += split_size + total_scene_count += len(scene_records) + + report = { + "dataset": Dataset.KITSCENES.value, + "dataset_revision": KITSCENES_SOURCE_REVISION, + "sdk_revision": KITSCENES_SDK_REVISION, + "schema_version": "kitscenes_benchmark_inventory_v1", + "splits": splits, + "total_scene_count": total_scene_count, + "total_size_bytes": total_size_bytes, + } + output_dir = Path("/tmp/kitscenes-benchmark-inventory") + output_dir.mkdir(parents=True, exist_ok=True) + output_path = output_dir / "inventory.json" + output_path.write_text( + json.dumps( + report, + allow_nan=False, + ensure_ascii=True, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="ascii", + ) + print( + "KITScenes benchmark inventory: " + f"scenes={total_scene_count} bytes={total_size_bytes}" + ) + return FlyteFile(os.fspath(output_path)) + + +@task( + container_image=DATA_PREP_IMAGE, + pod_template=_data_prep_pod_template(), + requests=Resources(cpu="2", mem="8Gi", ephemeral_storage="20Gi"), + limits=Resources(cpu="2", mem="8Gi", ephemeral_storage="20Gi"), +) +def create_kitscenes_paper_approximation_manifest( + val_shards: List[FlyteDirectory], + overlap_shards: List[FlyteDirectory], + release_id: str = "autoe2e-paper-approx-v1", +) -> KITScenesBenchmarkManifestOutput: + """Create a deterministic 200-window development manifest. + + The exact authority-issued sample UIDs are not public. This task scans only + immutable packed metadata, removes overlapping windows per scene, and ranks + candidates by a pinned hash seed. It never reads trajectory values or model + metrics while selecting samples. + """ + import hashlib + import json + import os + import tarfile + + from evaluation.kitscenes_benchmark import ( + KITScenesBenchmarkCandidate, + MANIFEST_SCHEMA_VERSION, + PAPER_APPROXIMATION_SELECTION_SEED, + PAPER_APPROXIMATION_SELECTION_VERSION, + PAPER_PROTOCOL_SOURCE, + PAPER_WINDOW_STEPS, + PROTOCOL_ID, + parse_benchmark_manifest, + sample_uid_digest, + select_paper_approximation_samples, + ) + from data_parsing.kit_scenes.source import KITSCENES_SDK_REVISION + + split_inputs = { + "val": (val_shards, "val"), + "overlap-train-val": ( + overlap_shards, + "overlap_train_val", + ), + } + candidates_by_split: dict[ + str, list[KITScenesBenchmarkCandidate] + ] = {} + packed_sources: dict[str, list[dict[str, object]]] = {} + empty_partition_ids: dict[str, list[str]] = {} + seen_partition_ids: set[str] = set() + for protocol_split, (shards, source_split) in split_inputs.items(): + if not shards: + raise ValueError( + f"KITScenes benchmark split {protocol_split} has no shards" + ) + split_candidates: list[KITScenesBenchmarkCandidate] = [] + split_sources: list[dict[str, object]] = [] + for shard in shards: + shard_uri = str( + getattr(shard, "remote_source", "") or shard + ) + shard_dir = Path(shard.download()) + packed_manifest_path = shard_dir / "manifest.json" + if not packed_manifest_path.is_file(): + raise FileNotFoundError( + "KITScenes benchmark shard has no manifest: " + f"{packed_manifest_path}" + ) + packed_manifest_bytes = packed_manifest_path.read_bytes() + packed_manifest = json.loads(packed_manifest_bytes) + expected_fields = { + "dataset": Dataset.KITSCENES.value, + "source_revision": KITSCENES_SOURCE_REVISION, + "source_split": source_split, + "data_role": "benchmark", + "dataset_version": KITSCENES_BENCHMARK_DATASET_VERSION, + "hz": 10, + "temporal_sampling": kitscenes_temporal_contract( + benchmark_protocol=True, + ), + } + for field, expected in expected_fields.items(): + actual = packed_manifest.get(field) + if actual != expected: + raise ValueError( + "KITScenes benchmark packed manifest differs from " + f"the evaluation contract: {field}={actual!r}, " + f"expected={expected!r}" + ) + partition_id = str( + packed_manifest.get("partition_id", "") + ) + if not partition_id: + raise ValueError( + "KITScenes benchmark shard has no partition_id" + ) + if partition_id in seen_partition_ids: + raise ValueError( + "KITScenes benchmark has duplicate partition_id " + f"{partition_id}" + ) + seen_partition_ids.add(partition_id) + + shard_names = list(packed_manifest.get("shard_names", [])) + shard_count = packed_manifest.get("shards") + total_samples = packed_manifest.get("total_samples") + shard_sample_counts = packed_manifest.get( + "shard_sample_counts" + ) + num_views = packed_manifest.get("num_views") + if ( + isinstance(shard_count, bool) + or not isinstance(shard_count, int) + or shard_count < 0 + ): + raise ValueError( + "KITScenes benchmark shard count must be a " + f"non-negative integer: {shard_count!r}" + ) + if ( + isinstance(total_samples, bool) + or not isinstance(total_samples, int) + or total_samples < 0 + ): + raise ValueError( + "KITScenes benchmark total_samples must be a " + f"non-negative integer: {total_samples!r}" + ) + if not isinstance(shard_sample_counts, dict): + raise ValueError( + "KITScenes benchmark shard_sample_counts must be a map" + ) + counted_samples = 0 + for shard_name, count in shard_sample_counts.items(): + if ( + not isinstance(shard_name, str) + or not shard_name + or isinstance(count, bool) + or not isinstance(count, int) + or count <= 0 + ): + raise ValueError( + "KITScenes benchmark shard_sample_counts contains " + f"an invalid entry: {shard_name!r}={count!r}" + ) + counted_samples += count + + is_empty = total_samples == 0 + if is_empty: + empty_contract = { + "num_views": 0, + "shards": 0, + "shard_names": [], + "shard_sample_counts": {}, + } + actual_empty_contract = { + "num_views": num_views, + "shards": shard_count, + "shard_names": shard_names, + "shard_sample_counts": shard_sample_counts, + } + if actual_empty_contract != empty_contract: + raise ValueError( + "KITScenes empty benchmark partition differs from " + f"the empty contract: {actual_empty_contract!r}" + ) + for field in ("has_map", "has_gps", "has_navigation"): + if bool(packed_manifest.get(field, False)): + raise ValueError( + "KITScenes empty benchmark partition requires " + f"{field}=false" + ) + empty_partition_ids.setdefault(protocol_split, []).append( + partition_id + ) + split_sources.append({ + "empty": True, + "manifest_sha256": hashlib.sha256( + packed_manifest_bytes + ).hexdigest(), + "partition_id": partition_id, + "sample_count": 0, + "uri": shard_uri, + }) + continue + + if num_views != 6: + raise ValueError( + "KITScenes benchmark packed manifest differs from " + "the evaluation contract: " + f"num_views={num_views!r}, expected=6" + ) + for field in ("has_map", "has_gps", "has_navigation"): + if not bool(packed_manifest.get(field, False)): + raise ValueError( + f"KITScenes benchmark shard requires {field}=true" + ) + if ( + not shard_names + or shard_count != len(shard_names) + or set(shard_names) != set(shard_sample_counts) + or counted_samples != total_samples + ): + raise ValueError( + "KITScenes benchmark non-empty partition has " + "inconsistent shard metadata: " + f"partition_id={partition_id!r}, shards={shard_count}, " + f"shard_names={len(shard_names)}, " + f"counted_samples={counted_samples}, " + f"total_samples={total_samples}" + ) + metadata_count = 0 + for shard_name in shard_names: + tar_path = shard_dir / str(shard_name) + if not tar_path.is_file(): + raise FileNotFoundError( + f"KITScenes packed tar is missing: {tar_path}" + ) + with tarfile.open(tar_path, "r") as archive: + for member in archive: + if not member.isfile() or not member.name.endswith( + ".meta.json" + ): + continue + stream = archive.extractfile(member) + if stream is None: + raise ValueError( + f"unable to read packed member {member.name}" + ) + metadata = json.loads(stream.read()) + sample_uid = str(metadata.get("sample_uid", "")) + split_group_uid = str( + metadata.get("split_group_uid", "") + ) + expected_prefix = "kitscenes-" + if not split_group_uid.startswith(expected_prefix): + raise ValueError( + "KITScenes sample has invalid split_group_uid " + f"{split_group_uid!r}" + ) + frame_index = metadata.get("frame_idx") + if ( + isinstance(frame_index, bool) + or not isinstance(frame_index, int) + ): + raise ValueError( + "KITScenes sample frame_idx must be an integer" + ) + split_candidates.append( + KITScenesBenchmarkCandidate( + sample_uid=sample_uid, + source_split=protocol_split, + scene_id=split_group_uid[ + len(expected_prefix): + ], + frame_index=frame_index, + ) + ) + metadata_count += 1 + if metadata_count != total_samples: + raise ValueError( + "KITScenes benchmark metadata count differs from " + f"manifest: {metadata_count} != {total_samples}" + ) + split_sources.append({ + "empty": False, + "manifest_sha256": hashlib.sha256( + packed_manifest_bytes + ).hexdigest(), + "partition_id": partition_id, + "sample_count": metadata_count, + "uri": shard_uri, + }) + candidates_by_split[protocol_split] = split_candidates + packed_sources[protocol_split] = sorted( + split_sources, + key=lambda item: str(item["partition_id"]), + ) + empty_partition_ids.setdefault(protocol_split, []) + empty_partition_ids[protocol_split].sort() + + sample_uids, selection = select_paper_approximation_samples( + candidates_by_split, + ) + empty_partition_count_by_split = { + split: len(empty_partition_ids[split]) + for split in sorted(empty_partition_ids) + } + temporal_sampling = kitscenes_temporal_contract( + benchmark_protocol=True, + ) + payload = { + "authority": "auto-e2e", + "benchmark_id": "autoe2e-kitscenes-paper-approx-v1", + "dataset_revision": KITSCENES_SOURCE_REVISION, + "frequency_hz": 10, + "history_adapter": "left_zero_pad_to_64", + "horizons_seconds": [3, 5], + "input_track": "camera-map-route", + "packed_sources": packed_sources, + "past_seconds": 4, + "protocol_id": PROTOCOL_ID, + "protocol_source": PAPER_PROTOCOL_SOURCE, + "protocol_status": "paper_protocol_approximation", + "release_id": release_id, + "sample_count": len(sample_uids), + "sample_uid_digest": sample_uid_digest(sample_uids), + "sample_uids": list(sample_uids), + "schema_version": MANIFEST_SCHEMA_VERSION, + "sdk_revision": KITSCENES_SDK_REVISION, + "selection": { + **selection, + "anchor_policy": ( + "first_packed_anchor_then_greedy_90_frame_stride" + ), + "empty_partition_count": sum( + empty_partition_count_by_split.values() ), - "guardrail/ade_relative_regression": guardrails["ade_m"][ - "relative_regression" + "empty_partition_count_by_split": ( + empty_partition_count_by_split + ), + "empty_partition_ids_by_split": { + split: empty_partition_ids[split] + for split in sorted(empty_partition_ids) + }, + "metric_or_target_values_read": False, + "packed_future_steps": temporal_sampling["abi_future_steps"], + "packed_history_steps": temporal_sampling["abi_history_steps"], + "paper_future_steps": 50, + "paper_observation_steps": 40, + "sampling_future_steps": temporal_sampling[ + "sampling_future_steps" ], - "guardrail/fde_relative_regression": guardrails["fde_m"][ - "relative_regression" + "sampling_history_steps": temporal_sampling[ + "sampling_history_steps" ], - "decision/supported": ( - 1.0 if decision["verdict"] == "supported" else 0.0 - ), - } - mlflow.log_metrics({ - key: float(value) - for key, value in metrics.items() - if value is not None - }) - mlflow.log_artifact(output_path) - - return FlyteFile(output_path) - - -@task( - container_image=EVAL_IMAGE, - requests=Resources(cpu="2", mem="8Gi", gpu="1"), - limits=Resources(cpu="2", mem="8Gi", gpu="1"), - environment={"MLFLOW_TRACKING_URI": MLFLOW_URI}, - pod_template=_large_shm_pod_template(), # /dev/shm for eval DataLoader workers (#121 P0) -) -def evaluate_rl_policy( - checkpoint: FlyteFile, - shards: List[FlyteDirectory], - train_metadata: FlyteFile, - dataset: Dataset = Dataset.L2D, -) -> EvalMetrics: - """Open-loop evaluation of the Offline-RL refined policy. - - Logs ADE/FDE, params (incl. rl/*), artifacts to the MLflow `offline-rl` - experiment and registers the refined checkpoint in the model registry. - """ - return _run_evaluation(checkpoint, shards, train_metadata, dataset, "offline-rl") + "selection_seed": PAPER_APPROXIMATION_SELECTION_SEED, + "selection_version": PAPER_APPROXIMATION_SELECTION_VERSION, + "window_steps": PAPER_WINDOW_STEPS, + }, + "source_splits": ["val", "overlap-train-val"], + } + parse_benchmark_manifest(payload) + output_dir = Path("/tmp/kitscenes-paper-approximation") + output_dir.mkdir(parents=True, exist_ok=True) + output_path = output_dir / "manifest.json" + output_path.write_text( + json.dumps( + payload, + allow_nan=False, + ensure_ascii=True, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="ascii", + ) + digest = hashlib.sha256(output_path.read_bytes()).hexdigest() + print( + "KITScenes paper approximation manifest: " + f"samples={len(sample_uids)} sha256={digest} " + f"selection={json.dumps(selection, sort_keys=True)}" + ) + return KITScenesBenchmarkManifestOutput( + manifest=FlyteFile(os.fspath(output_path)), + manifest_sha256=digest, + ) @task( @@ -7268,9 +10199,17 @@ def evaluate_kitscenes_benchmark_checkpoint( if not isinstance(payload["config"], dict): raise ValueError("benchmark checkpoint config must be an object") config = dict(payload["config"]) - training_policy = training_policy_from_config( - config, - Dataset.KITSCENES.value, + simple_xy_objective = ( + config.get("training_objective_version") + == SIMPLE_XY_IMITATION_OBJECTIVE_VERSION + ) + training_policy = ( + None + if simple_xy_objective + else training_policy_from_config( + config, + Dataset.KITSCENES.value, + ) ) epoch = int(payload["epoch"]) if epoch <= 0: @@ -7297,6 +10236,16 @@ def evaluate_kitscenes_benchmark_checkpoint( shard_identities: list[dict] = [] dataset_versions: set[str] = set() contract_digests: set[str] = set() + packed_source_splits: set[str] = set() + protocol_to_packed_split = { + "val": "val", + "overlap-train-val": "overlap_train_val", + } + expected_packed_splits = { + protocol_to_packed_split[split] + for split in manifest.source_splits + if split in protocol_to_packed_split + } for shard in benchmark_shards: shard_uri = str( getattr(shard, "remote_source", "") or shard @@ -7330,6 +10279,21 @@ def evaluate_kitscenes_benchmark_checkpoint( f"manifest: shard={packed_manifest.get('source_revision')!r} " f"manifest={manifest.dataset_revision!r}" ) + packed_source_split = str( + packed_manifest.get("source_split", "") + ) + if packed_source_split not in expected_packed_splits: + raise ValueError( + "benchmark shard source split differs from the fixed " + f"manifest: shard={packed_source_split!r} " + f"manifest={sorted(expected_packed_splits)!r}" + ) + if packed_manifest.get("data_role") != "benchmark": + raise ValueError( + "benchmark evaluation refuses shards not prepared with " + "data_role='benchmark'" + ) + packed_source_splits.add(packed_source_split) if int(packed_manifest.get("hz", 0)) != manifest.frequency_hz: raise ValueError( "benchmark shard frequency differs from the protocol: " @@ -7364,6 +10328,7 @@ def evaluate_kitscenes_benchmark_checkpoint( packed_manifest.get("shard_names", []) ), "source_revision": packed_manifest.get("source_revision"), + "source_split": packed_source_split, "total_samples": total_samples, "uri": shard_uri, }) @@ -7403,6 +10368,12 @@ def evaluate_kitscenes_benchmark_checkpoint( ) if len(contract_digests) != 1: raise ValueError("benchmark shards mix packing contracts") + if packed_source_splits != expected_packed_splits: + raise ValueError( + "benchmark shards do not cover the fixed manifest splits: " + f"actual={sorted(packed_source_splits)} " + f"expected={sorted(expected_packed_splits)}" + ) shard_identities.sort( key=lambda item: ( str(item["partition_id"]), @@ -7485,9 +10456,13 @@ def evaluate_kitscenes_benchmark_checkpoint( "benchmark GPS trajectory has unexpected shape " f"{getattr(gps_future, 'shape', None)}" ) - policy_history = adapt_egomotion_history( - history, - training_policy, + policy_history = ( + history + if training_policy is None + else adapt_egomotion_history( + history, + training_policy, + ) ) limited_history = limit_egomotion_history( policy_history, @@ -8027,6 +11002,216 @@ def audit_kitscenes_target_reconstruction( # ============================================================ # Workflows # ============================================================ +@workflow +def wf_acquire_nuplan_raw_snapshot( + source_manifest: FlyteFile, + datasets_bucket: str, + aws_region: str = "us-west-2", + concurrency: int = 4, +) -> NuPlanRawSnapshotOutput: + """Acquire authorized nuPlan archives once into an immutable S3 snapshot.""" + return _acquire_nuplan_raw_snapshot( + source_manifest=source_manifest, + datasets_bucket=datasets_bucket, + aws_region=aws_region, + concurrency=concurrency, + ) + + +@workflow +def wf_pack_nuplan_reactive_dataset( + data_root: FlyteDirectory, + map_root: FlyteDirectory, + sensor_root: FlyteDirectory, + db_files: List[str], + source_revision: str, + map_version: str, + limit_total_scenarios: int = 0, + image_size: int = 256, + samples_per_shard: int = 1000, + max_rejection_fraction: float = 0.0, +) -> FlyteDirectory: + """Build the immutable Stage A source shards from raw nuPlan assets.""" + return pack_nuplan_reactive_dataset( + data_root=data_root, + map_root=map_root, + sensor_root=sensor_root, + db_files=db_files, + source_revision=source_revision, + map_version=map_version, + limit_total_scenarios=limit_total_scenarios, + image_size=image_size, + samples_per_shard=samples_per_shard, + max_rejection_fraction=max_rejection_fraction, + ) + + +@workflow +def wf_train_reactive_nuplan_l2d( + nuplan_shards: List[FlyteDirectory], + l2d_shards: List[FlyteDirectory], + backbone: Backbone = Backbone.SWIN_V2_TINY, + stage_a_epochs: int = 3, + stage_b_epochs: int = 3, + batch_size: int = 2, + stage_a_lr: float = 1e-4, + stage_b_lr: float = 3e-5, + val_fraction: float = 0.1, + num_workers: int = 0, + training_seed: int = 149, + bev_weight: float = 1.0, + route_weight: float = 1.0, +) -> ReactiveTrainingProgramOutput: + """Run Stage A nuPlan and Stage B L2D with a weights-only boundary.""" + stage_a = train_reactive_multitask_stage( + shards=nuplan_shards, + dataset=Dataset.NUPLAN, + stage="nuplan_full", + parent_checkpoint=None, + backbone=backbone, + epochs=stage_a_epochs, + batch_size=batch_size, + lr=stage_a_lr, + val_fraction=val_fraction, + num_workers=num_workers, + training_seed=training_seed, + bev_weight=bev_weight, + route_weight=route_weight, + ) + stage_b = train_reactive_multitask_stage( + shards=l2d_shards, + dataset=Dataset.L2D, + stage="l2d_continuation", + parent_checkpoint=stage_a.checkpoint, + backbone=backbone, + epochs=stage_b_epochs, + batch_size=batch_size, + lr=stage_b_lr, + val_fraction=val_fraction, + num_workers=num_workers, + training_seed=training_seed, + bev_weight=0.0, + route_weight=route_weight, + ) + retention = evaluate_reactive_transfer_matrix( + stage_a_checkpoint=stage_a.checkpoint, + stage_b_checkpoint=stage_b.checkpoint, + nuplan_shards=nuplan_shards, + l2d_shards=l2d_shards, + batch_size=batch_size, + val_fraction=val_fraction, + num_workers=num_workers, + ) + return ReactiveTrainingProgramOutput( + stage_a_checkpoint=stage_a.checkpoint, + stage_a_metadata=stage_a.metadata, + stage_b_checkpoint=stage_b.checkpoint, + stage_b_metadata=stage_b.metadata, + retention_report=retention.report, + retention_report_sha256=retention.report_sha256, + ) + + +@workflow +def wf_benchmark_reactive_program( + stage_a_checkpoint: FlyteFile, + stage_b_checkpoint: FlyteFile, + benchmark_shards: List[FlyteDirectory], + benchmark_manifest: FlyteFile, + expected_manifest_sha256: str = "", + stage_a_mlflow_run_id: str = "", + stage_b_mlflow_run_id: str = "", + batch_size: int = 4, +) -> ReactiveBenchmarkProgramOutput: + """Evaluate predeclared Stage A/B checkpoints without optimizer access.""" + stage_a = evaluate_kitscenes_benchmark_checkpoint( + checkpoint=stage_a_checkpoint, + benchmark_shards=benchmark_shards, + benchmark_manifest=benchmark_manifest, + expected_manifest_sha256=expected_manifest_sha256, + mlflow_run_id=stage_a_mlflow_run_id, + batch_size=batch_size, + ) + stage_b = evaluate_kitscenes_benchmark_checkpoint( + checkpoint=stage_b_checkpoint, + benchmark_shards=benchmark_shards, + benchmark_manifest=benchmark_manifest, + expected_manifest_sha256=expected_manifest_sha256, + mlflow_run_id=stage_b_mlflow_run_id, + batch_size=batch_size, + ) + return ReactiveBenchmarkProgramOutput( + stage_a_ade_3s=stage_a.ade_3s, + stage_a_fde_3s=stage_a.fde_3s, + stage_a_ade_5s=stage_a.ade_5s, + stage_a_fde_5s=stage_a.fde_5s, + stage_a_predictions=stage_a.predictions, + stage_a_report=stage_a.report, + stage_b_ade_3s=stage_b.ade_3s, + stage_b_fde_3s=stage_b.fde_3s, + stage_b_ade_5s=stage_b.ade_5s, + stage_b_fde_5s=stage_b.fde_5s, + stage_b_predictions=stage_b.predictions, + stage_b_report=stage_b.report, + ) + + +@workflow +def wf_precompute_semantic_occupancy( + checkpoint: FlyteFile, + shard_dirs: List[FlyteDirectory], + dataset: str, + dataset_version: str, + dataset_manifest_sha256: str, + artifacts_bucket: str, + publication_timestamp: str, + repository_revision: str, + aws_region: str = "us-west-2", + batch_size: int = 2, + num_workers: int = 0, +) -> SemanticOccupancyPrecomputeOutput: + """Publish Dashboard semantic bodies without running model inference in API.""" + return precompute_semantic_occupancy_artifacts( + checkpoint=checkpoint, + shard_dirs=shard_dirs, + dataset=dataset, + dataset_version=dataset_version, + dataset_manifest_sha256=dataset_manifest_sha256, + artifacts_bucket=artifacts_bucket, + publication_timestamp=publication_timestamp, + repository_revision=repository_revision, + aws_region=aws_region, + batch_size=batch_size, + num_workers=num_workers, + ) + + +@workflow +def wf_precompute_bevformer_v2_occupancy( + checkpoint: FlyteFile, + shard_dirs: List[FlyteDirectory], + dataset: str, + dataset_version: str, + dataset_manifest_sha256: str, + artifacts_bucket: str, + publication_timestamp: str, + aws_region: str = "us-west-2", + score_threshold: float = 0.2, +) -> SemanticOccupancyPrecomputeOutput: + """Publish official V2 detection footprints for the Occupancy Dashboard.""" + return precompute_bevformer_v2_occupancy_artifacts( + checkpoint=checkpoint, + shard_dirs=shard_dirs, + dataset=dataset, + dataset_version=dataset_version, + dataset_manifest_sha256=dataset_manifest_sha256, + artifacts_bucket=artifacts_bucket, + publication_timestamp=publication_timestamp, + aws_region=aws_region, + score_threshold=score_threshold, + ) + + @workflow def wf_evaluate_kitscenes_benchmark( checkpoint: FlyteFile, @@ -8230,6 +11415,10 @@ def _map_dataset_partitions( ingest_concurrency: int, label_concurrency: int, pack_concurrency: int, + reactive_targets: bool, + osm_graph_snapshot: Optional[FlyteFile], + source_split: str, + data_role: str, ) -> List[FlyteDirectory]: """Execute each data-prep stage as one bounded Flyte array node.""" for name, value in ( @@ -8239,6 +11428,18 @@ def _map_dataset_partitions( ): if value <= 0: raise ValueError(f"{name} must be positive, got {value}") + if ( + reactive_targets + and dataset == Dataset.L2D + and osm_graph_snapshot is None + ): + raise ValueError( + "L2D Reactive packing requires an OSM graph snapshot" + ) + if reasoning_teacher != "none" and data_role != "training": + raise ValueError( + "benchmark dataset preparation must not invoke reasoning teachers" + ) ingest = map_task( functools.partial( @@ -8246,6 +11447,8 @@ def _map_dataset_partitions( dataset=dataset, source_revision=source_revision, episodes=0, + source_split=source_split, + data_role=data_role, ), concurrency=ingest_concurrency, ) @@ -8278,6 +11481,10 @@ def _map_dataset_partitions( episodes=0, world_model=True, expected_reasoning_label_count=None, + reactive_targets=reactive_targets, + osm_graph_snapshot=osm_graph_snapshot, + source_split=source_split, + data_role=data_role, ), concurrency=pack_concurrency, ) @@ -8299,6 +11506,10 @@ def _map_dataset_partitions( world_model=world_model, reasoning_labels=None, expected_reasoning_label_count=None, + reactive_targets=reactive_targets, + osm_graph_snapshot=osm_graph_snapshot, + source_split=source_split, + data_role=data_role, ), concurrency=pack_concurrency, ) @@ -8469,6 +11680,8 @@ def wf_create_dataset_sharded( ingest_concurrency: int = 60, label_concurrency: int = 5, pack_concurrency: int = 60, + reactive_targets: bool = False, + osm_graph_snapshot: Optional[FlyteFile] = None, ) -> List[FlyteDirectory]: """Fan out immutable source groups through bounded ingest/label/pack arrays. @@ -8502,6 +11715,187 @@ def wf_create_dataset_sharded( ingest_concurrency=ingest_concurrency, label_concurrency=label_concurrency, pack_concurrency=pack_concurrency, + reactive_targets=reactive_targets, + osm_graph_snapshot=osm_graph_snapshot, + source_split="train", + data_role="training", + ) + + +@workflow +def wf_audit_kitscenes_benchmark_inventory() -> FlyteFile: + """Report pinned held-out archive sizes before any large download.""" + return audit_kitscenes_benchmark_inventory() + + +@workflow +def wf_prepare_kitscenes_paper_approximation( + val_scene_limit: int = 0, + overlap_scene_limit: int = 0, + ingest_concurrency: int = 20, + pack_concurrency: int = 20, + release_id: str = "autoe2e-paper-approx-v1", +) -> KITScenesBenchmarkPreparationOutput: + """Pack held-out scenes and freeze a deterministic 200-window manifest. + + This workflow is evaluation-only. It cannot select ``train`` scenes and + does not invoke any optimizer, reasoning teacher, or checkpoint selection. + A zero scene limit means all official scenes in that split. + """ + val_partitions = plan_fanout_partitions( + dataset=Dataset.KITSCENES, + source_revision=KITSCENES_SOURCE_REVISION, + episodes=val_scene_limit, + start_ep=-1, + end_ep=-1, + partition_size=1, + max_partitions=200, + max_missing_scenes=0, + split="val", + data_role="benchmark", + ) + overlap_partitions = plan_fanout_partitions( + dataset=Dataset.KITSCENES, + source_revision=KITSCENES_SOURCE_REVISION, + episodes=overlap_scene_limit, + start_ep=-1, + end_ep=-1, + partition_size=1, + max_partitions=200, + max_missing_scenes=0, + split="overlap_train_val", + data_role="benchmark", + ) + val_shards = _map_dataset_partitions( + partitions=val_partitions, + dataset=Dataset.KITSCENES, + source_revision=KITSCENES_SOURCE_REVISION, + dataset_version=KITSCENES_BENCHMARK_DATASET_VERSION, + image_size=256, + world_model=False, + reasoning_teacher="none", + prompt_version="unused", + label_stride=10, + label_workers=1, + ingest_concurrency=ingest_concurrency, + label_concurrency=1, + pack_concurrency=pack_concurrency, + reactive_targets=False, + osm_graph_snapshot=None, + source_split="val", + data_role="benchmark", + ) + overlap_shards = _map_dataset_partitions( + partitions=overlap_partitions, + dataset=Dataset.KITSCENES, + source_revision=KITSCENES_SOURCE_REVISION, + dataset_version=KITSCENES_BENCHMARK_DATASET_VERSION, + image_size=256, + world_model=False, + reasoning_teacher="none", + prompt_version="unused", + label_stride=10, + label_workers=1, + ingest_concurrency=ingest_concurrency, + label_concurrency=1, + pack_concurrency=pack_concurrency, + reactive_targets=False, + osm_graph_snapshot=None, + source_split="overlap_train_val", + data_role="benchmark", + ) + manifest = create_kitscenes_paper_approximation_manifest( + val_shards=val_shards, + overlap_shards=overlap_shards, + release_id=release_id, + ) + return KITScenesBenchmarkPreparationOutput( + val_shards=val_shards, + overlap_shards=overlap_shards, + manifest=manifest.manifest, + manifest_sha256=manifest.manifest_sha256, + ) + + +@workflow +def wf_build_l2d_osm_graph_artifact( + source_pbf: FlyteFile, + source_revision: str, + source_date: str, + attribution: str = "OpenStreetMap contributors", +) -> FlyteFile: + """Build the immutable OSM graph used by all L2D pack partitions.""" + return build_l2d_osm_graph_artifact( + source_pbf=source_pbf, + source_revision=source_revision, + source_date=source_date, + attribution=attribution, + ) + + +@workflow +def wf_pack_l2d_reactive_dataset( + osm_graph_snapshot: FlyteFile, + episodes: int = 0, + start_ep: int = -1, + end_ep: int = -1, + partition_size: int = 1, + max_partitions: int = 600, + ingest_concurrency: int = 40, + pack_concurrency: int = 40, +) -> List[FlyteDirectory]: + """Repack L2D with trajectory, OSM Map, and Route targets.""" + return wf_create_dataset_sharded( + dataset=Dataset.L2D, + source_revision=L2D_SOURCE_REVISION, + dataset_version=L2D_REACTIVE_DATASET_VERSION, + episodes=episodes, + start_ep=start_ep, + end_ep=end_ep, + partition_size=partition_size, + image_size=256, + world_model=False, + reasoning_teacher="none", + max_partitions=max_partitions, + max_missing_scenes=0, + ingest_concurrency=ingest_concurrency, + label_concurrency=1, + pack_concurrency=pack_concurrency, + reactive_targets=True, + osm_graph_snapshot=osm_graph_snapshot, + ) + + +@workflow +def wf_prepare_l2d_reactive_dataset( + source_pbf: FlyteFile, + source_revision: str, + source_date: str, + attribution: str = "OpenStreetMap contributors", + episodes: int = 0, + start_ep: int = -1, + end_ep: int = -1, + partition_size: int = 1, + max_partitions: int = 600, + ingest_concurrency: int = 40, + pack_concurrency: int = 40, +) -> List[FlyteDirectory]: + """Build a pinned OSM graph and pack one immutable L2D subset.""" + osm_graph_snapshot = build_l2d_osm_graph_artifact( + source_pbf=source_pbf, + source_revision=source_revision, + source_date=source_date, + attribution=attribution, + ) + return wf_pack_l2d_reactive_dataset( + osm_graph_snapshot=osm_graph_snapshot, + episodes=episodes, + start_ep=start_ep, + end_ep=end_ep, + partition_size=partition_size, + max_partitions=max_partitions, + ingest_concurrency=ingest_concurrency, + pack_concurrency=pack_concurrency, ) @@ -8893,7 +12287,7 @@ def wf_precompute_overlays( dataset_version: str = DATASET_PACK_VERSION, dynamo_table: str = "auto-e2e-console", aws_region: str = "us-west-2", - base_seeds: List[int] = [0], + base_seeds: Optional[List[int]] = None, batch_size: int = 32, num_workers: int = 4, sampler: str = "model-default", @@ -8912,6 +12306,11 @@ def wf_precompute_overlays( prepare_overlay_set, resolve_overlay_model, ) + normalized_base_seeds = ( + [0] + if base_seeds is None + else base_seeds + ) resolved = resolve_overlay_model( registered_model_name=registered_model_name, @@ -8929,7 +12328,7 @@ def wf_precompute_overlays( artifacts_bucket=artifacts_bucket, dynamo_table=dynamo_table, aws_region=aws_region, - base_seeds=base_seeds, + base_seeds=normalized_base_seeds, sampler=sampler, ) result = precompute_overlay_partition( @@ -8946,7 +12345,7 @@ def wf_precompute_overlays( artifacts_bucket=artifacts_bucket, dynamo_table=dynamo_table, aws_region=aws_region, - base_seeds=base_seeds, + base_seeds=normalized_base_seeds, batch_size=batch_size, num_workers=num_workers, sampler=sampler, @@ -9025,7 +12424,7 @@ def wf_publish_and_precompute_overlays( dataset_version: str = DATASET_PACK_VERSION, dynamo_table: str = "auto-e2e-console", aws_region: str = "us-west-2", - base_seeds: List[int] = [0], + base_seeds: Optional[List[int]] = None, batch_size: int = 32, num_workers: int = 4, copy_workers: int = 16, @@ -9085,7 +12484,7 @@ def wf_publish_full_run_overlays( dataset_version: str = DATASET_PACK_VERSION, dynamo_table: str = "auto-e2e-console", aws_region: str = "us-west-2", - base_seeds: List[int] = [0], + base_seeds: Optional[List[int]] = None, batch_size: int = 32, num_workers: int = 4, copy_workers: int = 16, @@ -9137,7 +12536,7 @@ def wf_publish_selected_checkpoint_overlays( dataset_version: str = DATASET_PACK_VERSION, dynamo_table: str = "auto-e2e-console", aws_region: str = "us-west-2", - base_seeds: List[int] = [0], + base_seeds: Optional[List[int]] = None, batch_size: int = 32, num_workers: int = 4, copy_workers: int = 16, @@ -9203,7 +12602,7 @@ def wf_create_publish_and_precompute_overlays( registered_model_name: str = "auto-e2e-driving-policy", dynamo_table: str = "auto-e2e-console", aws_region: str = "us-west-2", - base_seeds: List[int] = [0], + base_seeds: Optional[List[int]] = None, batch_size: int = 32, num_workers: int = 4, copy_workers: int = 16, @@ -9256,7 +12655,7 @@ def wf_export_trajectory_report( dataset_manifest: FlyteFile, overlay_manifest: FlyteFile, selection_manifest: Optional[FlyteFile] = None, - scene_uids: List[str] = [], + scene_uids: Optional[List[str]] = None, seed_index: int = 0, camera_index: int = 0, max_frames_per_scene: int = 300, From 46ebae82daab46f87a1c70d5c454e96fc512636e Mon Sep 17 00:00:00 2001 From: riita10069 Date: Sat, 22 Aug 2026 01:09:52 +0900 Subject: [PATCH 02/15] feat(occupancy): adapt HENet segmentation to ASOC geometry Preserve physical coordinates so HENet outputs remain comparable in the Dashboard. Signed-off-by: riita10069 --- Platform/pipelines/henet_occupancy.py | 294 ++++++++++++++++++++++++++ 1 file changed, 294 insertions(+) create mode 100644 Platform/pipelines/henet_occupancy.py diff --git a/Platform/pipelines/henet_occupancy.py b/Platform/pipelines/henet_occupancy.py new file mode 100644 index 000000000..0c38e96b7 --- /dev/null +++ b/Platform/pipelines/henet_occupancy.py @@ -0,0 +1,294 @@ +"""Pure HENet BEV segmentation adaptation for ASOC publications. + +HENet publishes three NuScenes BEV segmentation probabilities on a 200 x 200 +grid with 0.5 metre cells. This module preserves their physical coordinates +when producing the Dashboard's 450 x 300 ASOC geometry. +""" + +from __future__ import annotations + +import math +from collections.abc import Mapping + +import numpy as np + +from navigation.geometry import ( + AUTOE2E_NAVIGATION_GEOMETRY, + NavigationRasterGeometry, +) + +HENET_REPOSITORY = "https://github.com/VDIGPKU/HENet" +HENET_REVISION = "29ca81dd109cabe0a0c53ee354c4a74ad1559740" +HENET_CONFIG_NAME = "henet_det_bevseg.py" +HENET_WEIGHT_SOURCE_URL = ( + "https://drive.google.com/drive/folders/" + "1AYajSnL1JLrFOTHcpjX7WlRRLXUxQhwV" +) +HENET_CODE_LICENSE_SPDX = "LicenseRef-HENet-Research-Only" +HENET_WEIGHT_LICENSE_SPDX = "LicenseRef-HENet-Research-Only" +HENET_TRAINING_DATA_LICENSE_SPDX = "NOASSERTION" +HENET_ARTIFACT_KIND = "native-semantic-occupancy" +HENET_HEAD_VERSION = "henet-det-bevseg-v1" +HENET_CAMERA_ORDER = (1, 0, 2, 4, 3, 5) +HENET_CAMERA_COUNT = len(HENET_CAMERA_ORDER) +HENET_SOURCE_HZ = 10 +HENET_MODEL_HZ = 2 +HENET_SHORT_FRAME_OFFSETS = (0, -5, -10) +HENET_LONG_FRAME_OFFSETS = tuple(range(0, -45, -5)) +HENET_INPUT_HEIGHT = 640 +HENET_INPUT_WIDTH = 1152 +HENET_LONGTERM_INPUT_HEIGHT = 256 +HENET_LONGTERM_INPUT_WIDTH = 704 +HENET_SOURCE_X_MIN_M = -50.0 +HENET_SOURCE_X_MAX_M = 50.0 +HENET_SOURCE_Y_MIN_M = -50.0 +HENET_SOURCE_Y_MAX_M = 50.0 +HENET_SOURCE_METERS_PER_CELL = 0.5 +HENET_SOURCE_HEIGHT = 200 +HENET_SOURCE_WIDTH = 200 +HENET_SEGMENTATION_CLASS_NAMES = ( + "vehicle", + "drivable_area", + "divider", +) + +SEMANTIC_OCCUPANCY_CLASS_NAMES = ( + "drivable_area", + "lane_area", + "intersection", + "crosswalk", + "stop_line", + "vehicle", + "vulnerable_road_user", + "other_obstacle", +) + + +def _validate_sha256(value: str, label: str) -> str: + if len(value) != 64 or any( + character not in "0123456789abcdef" + for character in value + ): + raise ValueError(f"{label} must be a lowercase SHA-256") + return value + + +def _rq_decomposition(matrix: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + """Return upper-triangular and orthonormal factors for a 3 x 3 matrix.""" + values = np.asarray(matrix, dtype=np.float64) + if values.shape != (3, 3) or not np.isfinite(values).all(): + raise ValueError("projection matrix must be a finite [3,3] matrix") + + # NumPy has QR but not RQ. Reverse the axes around QR to obtain M = K @ R. + orthonormal, upper = np.linalg.qr(np.flipud(values).T) + intrinsic = np.flipud(upper.T) + intrinsic = np.fliplr(intrinsic) + rotation = orthonormal.T + rotation = np.flipud(rotation) + return intrinsic, rotation + + +def decompose_pinhole_projection( + projection_ref_to_camera: np.ndarray, +) -> tuple[np.ndarray, np.ndarray]: + """Recover packed-image intrinsics and camera-to-ego from ``K[R|t]``. + + The packed KITScenes calibration uses the top-lidar FLU frame as its + reference. HENet's ``sensor2ego`` tensor is therefore camera-to-top-lidar. + """ + projection = np.asarray(projection_ref_to_camera, dtype=np.float64) + if projection.shape != (3, 4) or not np.isfinite(projection).all(): + raise ValueError("projection must be a finite [3,4] matrix") + + intrinsic, rotation_ref_to_camera = _rq_decomposition(projection[:, :3]) + if abs(intrinsic[2, 2]) < 1e-12: + raise ValueError("projection intrinsic scale is singular") + + signs = np.where(np.diag(intrinsic) < 0.0, -1.0, 1.0) + sign_matrix = np.diag(signs) + intrinsic = intrinsic @ sign_matrix + rotation_ref_to_camera = sign_matrix @ rotation_ref_to_camera + if np.linalg.det(rotation_ref_to_camera) < 0.0: + intrinsic[:, 2] *= -1.0 + rotation_ref_to_camera[2, :] *= -1.0 + if not np.allclose( + rotation_ref_to_camera @ rotation_ref_to_camera.T, + np.eye(3), + rtol=0.0, + atol=1e-6, + ): + raise ValueError("projection rotation is not orthonormal") + + intrinsic /= intrinsic[2, 2] + translation_ref_to_camera = np.linalg.solve( + intrinsic, + projection[:, 3], + ) + camera_to_ref = np.eye(4, dtype=np.float64) + camera_to_ref[:3, :3] = rotation_ref_to_camera.T + camera_to_ref[:3, 3] = ( + -rotation_ref_to_camera.T @ translation_ref_to_camera + ) + return intrinsic, camera_to_ref + + +def _source_coordinates( + x_forward_m: np.ndarray, + y_left_m: np.ndarray, +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + source_row = ( + (x_forward_m - HENET_SOURCE_X_MIN_M) + / HENET_SOURCE_METERS_PER_CELL + - 0.5 + ) + source_col = ( + (y_left_m - HENET_SOURCE_Y_MIN_M) + / HENET_SOURCE_METERS_PER_CELL + - 0.5 + ) + valid = ( + (source_row >= 0.0) + & (source_row <= HENET_SOURCE_HEIGHT - 1) + & (source_col >= 0.0) + & (source_col <= HENET_SOURCE_WIDTH - 1) + ) + return source_row, source_col, valid + + +def _sample_henet_probability( + probability: np.ndarray, + *, + x_forward_m: np.ndarray, + y_left_m: np.ndarray, +) -> np.ndarray: + """Bilinearly sample HENet cells at Dashboard pixel centres.""" + source_row, source_col, valid = _source_coordinates( + x_forward_m, + y_left_m, + ) + row0 = np.floor(source_row).astype(np.int64) + col0 = np.floor(source_col).astype(np.int64) + row1 = np.minimum(row0 + 1, HENET_SOURCE_HEIGHT - 1) + col1 = np.minimum(col0 + 1, HENET_SOURCE_WIDTH - 1) + row0 = np.clip(row0, 0, HENET_SOURCE_HEIGHT - 1) + col0 = np.clip(col0, 0, HENET_SOURCE_WIDTH - 1) + row_fraction = source_row - np.floor(source_row) + col_fraction = source_col - np.floor(source_col) + + sampled = np.zeros( + (probability.shape[0], *x_forward_m.shape), + dtype=np.float32, + ) + for class_index, source in enumerate(probability): + top = ( + source[row0, col0] * (1.0 - col_fraction) + + source[row0, col1] * col_fraction + ) + bottom = ( + source[row1, col0] * (1.0 - col_fraction) + + source[row1, col1] * col_fraction + ) + sampled[class_index] = np.where( + valid, + top * (1.0 - row_fraction) + bottom * row_fraction, + 0.0, + ) + return sampled + + +def adapt_henet_segmentation( + probability: np.ndarray, + *, + geometry: NavigationRasterGeometry = AUTOE2E_NAVIGATION_GEOMETRY, +) -> np.ndarray: + """Map official HENet probabilities to the fixed ASOC taxonomy and grid. + + HENet's third output is a road divider rather than a lane area. It is + retained in ASOC's lane channel only to make the three published semantic + outputs inspectable in one fixed Dashboard taxonomy. + """ + values = np.asarray(probability, dtype=np.float32) + expected_shape = ( + len(HENET_SEGMENTATION_CLASS_NAMES), + HENET_SOURCE_HEIGHT, + HENET_SOURCE_WIDTH, + ) + if values.shape != expected_shape: + raise ValueError( + "HENet probability must have shape " + f"{list(expected_shape)}, got {list(values.shape)}" + ) + if not np.isfinite(values).all() or np.any(values < 0.0) or np.any( + values > 1.0 + ): + raise ValueError("HENet probability must be finite in [0,1]") + + x_forward_m, y_left_m = geometry.pixel_center_grids() + sampled = _sample_henet_probability( + values, + x_forward_m=x_forward_m, + y_left_m=y_left_m, + ) + output = np.zeros( + ( + len(SEMANTIC_OCCUPANCY_CLASS_NAMES), + geometry.height_px, + geometry.width_px, + ), + dtype=np.float32, + ) + output[SEMANTIC_OCCUPANCY_CLASS_NAMES.index("drivable_area")] = sampled[ + HENET_SEGMENTATION_CLASS_NAMES.index("drivable_area") + ] + output[SEMANTIC_OCCUPANCY_CLASS_NAMES.index("lane_area")] = sampled[ + HENET_SEGMENTATION_CLASS_NAMES.index("divider") + ] + output[SEMANTIC_OCCUPANCY_CLASS_NAMES.index("vehicle")] = sampled[ + HENET_SEGMENTATION_CLASS_NAMES.index("vehicle") + ] + return output + + +def provenance(checkpoint_sha256: str) -> Mapping[str, object]: + """Return the immutable scientific provenance for one HENet checkpoint.""" + return { + "artifact_kind": HENET_ARTIFACT_KIND, + "config": HENET_CONFIG_NAME, + "head_version": HENET_HEAD_VERSION, + "repository": HENET_REPOSITORY, + "repository_revision": HENET_REVISION, + "weight_sha256": _validate_sha256( + checkpoint_sha256, + "checkpoint_sha256", + ), + "weight_source_url": HENET_WEIGHT_SOURCE_URL, + "code_license_spdx": HENET_CODE_LICENSE_SPDX, + "weight_license_spdx": HENET_WEIGHT_LICENSE_SPDX, + "training_data_license_spdx": HENET_TRAINING_DATA_LICENSE_SPDX, + "supported_semantic_classes": [ + "drivable_area", + "lane_area", + "vehicle", + ], + "teacher_available": False, + "limitations": [ + ( + "The official HENet repository and supplied checkpoint are " + "free only for academic research; commercial use requires " + "authorization from the authors." + ), + ( + "The lane channel contains HENet road-divider probability, " + "not a lane-area segmentation." + ), + ( + "KITScenes 256-square camera images are upsampled into the " + "official HENet input sizes and therefore have lower source " + "resolution than official nuScenes evaluation." + ), + ( + "Teacher and Error views are unavailable because KITScenes " + "packed shards do not contain perception ground truth." + ), + ], + } From b940bc3a10498fd5035b99fd46398326763aa20f Mon Sep 17 00:00:00 2001 From: riita10069 Date: Sat, 22 Aug 2026 01:11:07 +0900 Subject: [PATCH 03/15] test(occupancy): cover HENet ASOC adaptation Lock calibration, taxonomy, and physical-grid conversion before wiring inference. Signed-off-by: riita10069 --- Model/tests/test_henet_occupancy.py | 158 ++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 Model/tests/test_henet_occupancy.py diff --git a/Model/tests/test_henet_occupancy.py b/Model/tests/test_henet_occupancy.py new file mode 100644 index 000000000..8b5311ac9 --- /dev/null +++ b/Model/tests/test_henet_occupancy.py @@ -0,0 +1,158 @@ +import numpy as np +import pytest + +from navigation.geometry import NavigationRasterGeometry +from Platform.pipelines.henet_occupancy import ( + HENET_CODE_LICENSE_SPDX, + HENET_SEGMENTATION_CLASS_NAMES, + adapt_henet_segmentation, + decompose_pinhole_projection, + provenance, +) + + +def _geometry() -> NavigationRasterGeometry: + return NavigationRasterGeometry( + geometry_id="henet-test-2x2-1m", + height_px=2, + width_px=2, + meters_per_pixel=1.0, + x_min_m=-1.0, + x_max_m=1.0, + y_min_m=-1.0, + y_max_m=1.0, + ego_anchor_row=0.5, + ego_anchor_col=0.5, + matching_pc_range=(-1.0, -1.0, -2.0, 1.0, 1.0, 2.0), + matching_bev_h=2, + matching_bev_w=2, + route_corridor_width_m=1.0, + destination_marker_radius_m=1.0, + route_rear_clip_m=1.0, + ) + + +def test_projection_decomposition_recovers_intrinsics_and_camera_pose(): + intrinsic = np.asarray( + [ + [800.0, 0.0, 128.0], + [0.0, 810.0, 129.0], + [0.0, 0.0, 1.0], + ] + ) + camera_to_ego = np.eye(4) + camera_to_ego[:3, :3] = np.asarray( + [ + [0.0, -1.0, 0.0], + [1.0, 0.0, 0.0], + [0.0, 0.0, 1.0], + ] + ) + camera_to_ego[:3, 3] = [1.2, -0.4, 1.5] + ego_to_camera = np.linalg.inv(camera_to_ego) + projection = intrinsic @ ego_to_camera[:3, :] + + recovered_intrinsic, recovered_camera_to_ego = ( + decompose_pinhole_projection(projection) + ) + + np.testing.assert_allclose(recovered_intrinsic, intrinsic, atol=1e-8) + np.testing.assert_allclose( + recovered_camera_to_ego, + camera_to_ego, + atol=1e-8, + ) + + +def test_adaptation_uses_physical_coordinates_and_fixed_taxonomy(): + source = np.zeros((3, 200, 200), dtype=np.float32) + source[ + HENET_SEGMENTATION_CLASS_NAMES.index("drivable_area") + ] = ( + np.arange(200, dtype=np.float32)[:, None] * 0.5 + 0.25 + ) / 100.0 + source[ + HENET_SEGMENTATION_CLASS_NAMES.index("divider") + ] = ( + np.arange(200, dtype=np.float32)[None, :] * 0.5 + 0.25 + ) / 100.0 + source[ + HENET_SEGMENTATION_CLASS_NAMES.index("vehicle") + ].fill(0.75) + + output = adapt_henet_segmentation(source, geometry=_geometry()) + + assert output.shape == (8, 2, 2) + np.testing.assert_allclose( + output[0], + [[0.505, 0.505], [0.495, 0.495]], + atol=1e-6, + ) + np.testing.assert_allclose( + output[1], + [[0.505, 0.495], [0.505, 0.495]], + atol=1e-6, + ) + np.testing.assert_allclose(output[5], 0.75, atol=1e-6) + assert not output[2:5].any() + assert not output[6:].any() + + +def test_adaptation_zeroes_dashboard_cells_outside_henet_extent(): + source = np.ones((3, 200, 200), dtype=np.float32) + geometry = NavigationRasterGeometry( + geometry_id="henet-test-2x2-50m", + height_px=2, + width_px=2, + meters_per_pixel=50.0, + x_min_m=-50.0, + x_max_m=50.0, + y_min_m=-50.0, + y_max_m=50.0, + ego_anchor_row=0.5, + ego_anchor_col=0.5, + matching_pc_range=(-50.0, -50.0, -2.0, 50.0, 50.0, 2.0), + matching_bev_h=2, + matching_bev_w=2, + route_corridor_width_m=1.0, + destination_marker_radius_m=1.0, + route_rear_clip_m=1.0, + ) + + output = adapt_henet_segmentation(source, geometry=geometry) + + assert np.all(output[[0, 1, 5]] == 1.0) + + +@pytest.mark.parametrize( + "source", + [ + np.zeros((3, 199, 200), dtype=np.float32), + np.full((3, 200, 200), np.nan, dtype=np.float32), + np.full((3, 200, 200), 1.1, dtype=np.float32), + ], +) +def test_adaptation_rejects_invalid_official_outputs(source): + with pytest.raises(ValueError): + adapt_henet_segmentation(source) + + +def test_provenance_discloses_research_only_license_and_divider_mapping(): + metadata = provenance("a" * 64) + + assert metadata["artifact_kind"] == "native-semantic-occupancy" + assert metadata["teacher_available"] is False + assert metadata["supported_semantic_classes"] == [ + "drivable_area", + "lane_area", + "vehicle", + ] + assert metadata["code_license_spdx"] == HENET_CODE_LICENSE_SPDX + assert any( + "road-divider" in limitation + for limitation in metadata["limitations"] + ) + assert any( + "academic research" in limitation + for limitation in metadata["limitations"] + ) From 612e4ebc8371a9b3b417fbf7be0541007da80795 Mon Sep 17 00:00:00 2001 From: riita10069 Date: Sat, 22 Aug 2026 01:12:59 +0900 Subject: [PATCH 04/15] feat(occupancy): run HENet on packed KITScenes frames Reconstruct the official multi-frame camera tensors from calibrated Dashboard shards. Signed-off-by: riita10069 --- Platform/pipelines/henet_runtime.py | 526 ++++++++++++++++++++++++++++ 1 file changed, 526 insertions(+) create mode 100644 Platform/pipelines/henet_runtime.py diff --git a/Platform/pipelines/henet_runtime.py b/Platform/pipelines/henet_runtime.py new file mode 100644 index 000000000..f245955b0 --- /dev/null +++ b/Platform/pipelines/henet_runtime.py @@ -0,0 +1,526 @@ +"""Official HENet inference over calibrated packed KITScenes samples.""" + +from __future__ import annotations + +import hashlib +import importlib +import io +import json +import subprocess +import sys +import tarfile +from collections import OrderedDict +from collections.abc import Iterable, Mapping, MutableMapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np + +from data_processing.geospatial import decode_pose +from Platform.pipelines.bevformer_v2_occupancy import ( + pose_to_world_from_top_lidar, +) +from Platform.pipelines.henet_occupancy import ( + HENET_CAMERA_COUNT, + HENET_CAMERA_ORDER, + HENET_CONFIG_NAME, + HENET_INPUT_HEIGHT, + HENET_INPUT_WIDTH, + HENET_LONGTERM_INPUT_HEIGHT, + HENET_LONGTERM_INPUT_WIDTH, + HENET_LONG_FRAME_OFFSETS, + HENET_REVISION, + HENET_SEGMENTATION_CLASS_NAMES, + HENET_SHORT_FRAME_OFFSETS, + adapt_henet_segmentation, + decompose_pinhole_projection, +) + +HENET_IMAGE_MEAN = (123.675, 116.28, 103.53) +HENET_IMAGE_STD = (58.395, 57.12, 57.375) + + +@dataclass(frozen=True) +class PackedHENetFrame: + """One six-camera KITScenes frame in HENet's official camera order.""" + + sample_uid: str + episode_id: str + frame_index: int + timestamp_ns: int + image_payloads: tuple[bytes, ...] + projection_ref_to_camera: np.ndarray + pose: Mapping[str, float | int] + + def __post_init__(self) -> None: + if not self.sample_uid or not self.episode_id: + raise ValueError("packed frame identity must not be empty") + if self.frame_index < 0: + raise ValueError("packed frame index must be non-negative") + if len(self.image_payloads) != HENET_CAMERA_COUNT: + raise ValueError("HENet requires six camera payloads") + if any(not payload for payload in self.image_payloads): + raise ValueError("packed camera payloads must not be empty") + projection = np.asarray(self.projection_ref_to_camera) + if projection.shape != (HENET_CAMERA_COUNT, 3, 4): + raise ValueError("packed projection must have shape [6,3,4]") + if not np.isfinite(projection).all(): + raise ValueError("packed projection must be finite") + + +def _sample_from_members( + sample_uid: str, + members: Mapping[str, bytes], +) -> PackedHENetFrame: + required = {"meta.json", "calib.json", "pose.npy"} + required.update( + f"cam_{camera}.jpg" + for camera in range(HENET_CAMERA_COUNT) + ) + missing = required - set(members) + if missing: + raise ValueError( + f"packed sample {sample_uid!r} is missing {sorted(missing)}" + ) + metadata = json.loads(members["meta.json"]) + calibration = json.loads(members["calib.json"]) + if not isinstance(metadata, Mapping) or not isinstance( + calibration, + Mapping, + ): + raise ValueError("packed metadata must be JSON objects") + if metadata.get("sample_uid") not in (None, sample_uid): + raise ValueError("packed sample UID differs from its tar key") + projection_spec = calibration.get("projection") + if ( + not isinstance(projection_spec, Mapping) + or projection_spec.get("type") != "pinhole" + or projection_spec.get("reference_frame") != "top_lidar_flu" + ): + raise ValueError("HENet requires top_lidar_flu pinhole calibration") + frame_index = metadata.get("frame_idx") + if isinstance(frame_index, bool) or not isinstance(frame_index, int): + raise ValueError("packed frame_idx must be an integer") + episode_id = metadata.get("split_group_uid") + if not isinstance(episode_id, str) or not episode_id: + raise ValueError("packed sample has no scene identity") + pose = decode_pose(members["pose.npy"]) + projection = np.asarray( + projection_spec.get("matrix"), + dtype=np.float64, + ) + if projection.shape != (HENET_CAMERA_COUNT, 3, 4): + raise ValueError("packed projection must have shape [6,3,4]") + camera_order = np.asarray(HENET_CAMERA_ORDER) + return PackedHENetFrame( + sample_uid=sample_uid, + episode_id=episode_id, + frame_index=frame_index, + timestamp_ns=int(pose["timestamp_ns"]), + image_payloads=tuple( + members[f"cam_{camera}.jpg"] + for camera in HENET_CAMERA_ORDER + ), + projection_ref_to_camera=projection[camera_order], + pose=pose, + ) + + +def iter_packed_henet_frames( + tar_path: str | Path, +) -> Iterable[PackedHENetFrame]: + """Yield packed KITScenes samples without decoding unrelated members.""" + current_key: str | None = None + current_members: dict[str, bytes] = {} + with tarfile.open(tar_path, mode="r:*") as archive: + for member in archive: + if not member.isfile() or "." not in member.name: + continue + sample_uid, suffix = member.name.split(".", 1) + if current_key is not None and sample_uid != current_key: + yield _sample_from_members(current_key, current_members) + current_members = {} + current_key = sample_uid + if suffix not in { + "meta.json", + "calib.json", + "pose.npy", + *( + f"cam_{camera}.jpg" + for camera in range(HENET_CAMERA_COUNT) + ), + }: + continue + stream = archive.extractfile(member) + if stream is None: + raise ValueError(f"could not read tar member {member.name!r}") + current_members[suffix] = stream.read() + if current_key is not None: + yield _sample_from_members(current_key, current_members) + + +def temporal_frames_for( + current: PackedHENetFrame, + history: Mapping[int, PackedHENetFrame], + *, + frame_offsets: Sequence[int], +) -> OrderedDict[int, PackedHENetFrame]: + """Select fixed official HENet inputs, repeating current at boundaries.""" + offsets = tuple(frame_offsets) + if not offsets or offsets[0] != 0 or any(offset > 0 for offset in offsets): + raise ValueError("HENet offsets must start at zero and use past frames") + if len(set(offsets)) != len(offsets): + raise ValueError("HENet frame offsets must be unique") + + selected: OrderedDict[int, PackedHENetFrame] = OrderedDict() + for offset in offsets: + candidate = ( + current + if offset == 0 + else history.get(current.frame_index + offset) + ) + if candidate is not None and candidate.episode_id != current.episode_id: + raise ValueError("temporal history crossed a KITScenes scene") + # The official NuScenes loader substitutes the key frame when a + # requested predecessor crosses a scene boundary. + selected[offset] = candidate if candidate is not None else current + return selected + + +def temporal_substitution_count( + current: PackedHENetFrame, + history: Mapping[int, PackedHENetFrame], + *, + frame_offsets: Sequence[int], +) -> int: + """Return how many fixed HENet temporal slots repeat the key frame.""" + return sum( + 1 + for offset in frame_offsets + if offset != 0 and current.frame_index + offset not in history + ) + + +def _world_pose( + frame: PackedHENetFrame, + *, + origin_latitude_deg: float, + origin_longitude_deg: float, +) -> np.ndarray: + return pose_to_world_from_top_lidar( + latitude_deg=float(frame.pose["latitude_deg"]), + longitude_deg=float(frame.pose["longitude_deg"]), + heading_deg_cw_from_north=float( + frame.pose["heading_deg_cw_from_north"] + ), + origin_latitude_deg=origin_latitude_deg, + origin_longitude_deg=origin_longitude_deg, + ) + + +def _preprocess_image( + payload: bytes, + *, + output_height: int, + output_width: int, +) -> tuple[Any, np.ndarray, np.ndarray]: + """Apply HENet's deterministic test resize, crop, and normalization.""" + import torch + from PIL import Image + + with Image.open(io.BytesIO(payload)) as source: + source = source.convert("RGB") + source_width, source_height = source.size + if source_width <= 0 or source_height <= 0: + raise ValueError("packed camera dimensions must be positive") + resize = output_width / source_width + resized_height = int(source_height * resize) + if resized_height < output_height: + raise ValueError("HENet test crop exceeds resized image height") + crop_top = resized_height - output_height + image = source.resize( + (output_width, resized_height), + resample=Image.Resampling.BICUBIC, + ).crop((0, crop_top, output_width, resized_height)) + + values = np.asarray(image, dtype=np.float32) + # HENet's mmlabNormalize receives a PIL RGB image with to_rgb=True, + # which reverses channels before applying the official RGB statistics. + values = values[:, :, ::-1] + values = (values - np.asarray(HENET_IMAGE_MEAN)) / np.asarray( + HENET_IMAGE_STD + ) + image_tensor = torch.from_numpy( + np.ascontiguousarray(values) + ).permute(2, 0, 1) + post_rotation = np.eye(3, dtype=np.float32) + post_rotation[0, 0] = resize + post_rotation[1, 1] = resize + post_translation = np.asarray([0.0, -crop_top, 0.0], dtype=np.float32) + return image_tensor, post_rotation, post_translation + + +def _image_inputs_for( + frames: Mapping[int, PackedHENetFrame], + *, + output_height: int, + output_width: int, + device: Any, +) -> tuple[Any, Any, Any, Any, Any, Any, Any]: + """Build a HENet img_inputs tuple with official tensor ordering.""" + import torch + + ordered = tuple(frames.values()) + if not ordered: + raise ValueError("HENet inference requires at least one frame") + current = ordered[0] + origin_latitude = float(current.pose["latitude_deg"]) + origin_longitude = float(current.pose["longitude_deg"]) + + per_frame_images = [] + sensor_to_ego = [] + ego_to_global = [] + intrinsics = [] + post_rotations = [] + post_translations = [] + for frame in ordered: + frame_images = [] + frame_sensor_to_ego = [] + frame_intrinsics = [] + frame_post_rotations = [] + frame_post_translations = [] + for payload, projection in zip( + frame.image_payloads, + frame.projection_ref_to_camera, + ): + image, post_rotation, post_translation = _preprocess_image( + payload, + output_height=output_height, + output_width=output_width, + ) + intrinsic, camera_to_ego = decompose_pinhole_projection( + projection + ) + frame_images.append(image) + frame_sensor_to_ego.append(camera_to_ego.astype(np.float32)) + frame_intrinsics.append(intrinsic.astype(np.float32)) + frame_post_rotations.append(post_rotation) + frame_post_translations.append(post_translation) + if len(frame_images) != HENET_CAMERA_COUNT: + raise ValueError("HENet frame has an unexpected camera count") + per_frame_images.append(torch.stack(frame_images)) + sensor_to_ego.extend(frame_sensor_to_ego) + intrinsics.extend(frame_intrinsics) + post_rotations.extend(frame_post_rotations) + post_translations.extend(frame_post_translations) + pose = _world_pose( + frame, + origin_latitude_deg=origin_latitude, + origin_longitude_deg=origin_longitude, + ).astype(np.float32) + ego_to_global.extend([pose] * HENET_CAMERA_COUNT) + + # HENet's image tensor is camera-major while its calibration tensors are + # frame-major. Both orders are required by BEVDet4D.prepare_inputs. + images = torch.stack(per_frame_images, dim=1).reshape( + HENET_CAMERA_COUNT * len(ordered), + 3, + output_height, + output_width, + ).unsqueeze(0) + sensor_to_ego_tensor = torch.from_numpy( + np.stack(sensor_to_ego) + ).unsqueeze(0) + ego_to_global_tensor = torch.from_numpy( + np.stack(ego_to_global) + ).unsqueeze(0) + intrinsics_tensor = torch.from_numpy(np.stack(intrinsics)).unsqueeze(0) + post_rotations_tensor = torch.from_numpy( + np.stack(post_rotations) + ).unsqueeze(0) + post_translations_tensor = torch.from_numpy( + np.stack(post_translations) + ).unsqueeze(0) + bda = torch.eye(3, dtype=torch.float32).unsqueeze(0) + return tuple( + tensor.to(device=device, non_blocking=True) + for tensor in ( + images, + sensor_to_ego_tensor, + ego_to_global_tensor, + intrinsics_tensor, + post_rotations_tensor, + post_translations_tensor, + bda, + ) + ) + + +def henet_inputs_for( + current: PackedHENetFrame, + history: Mapping[int, PackedHENetFrame], + *, + device: Any, +) -> tuple[tuple[Any, ...], tuple[Any, ...]]: + """Build the short-term and long-term official HENet img_inputs tuples.""" + short_frames = temporal_frames_for( + current, + history, + frame_offsets=HENET_SHORT_FRAME_OFFSETS, + ) + long_frames = temporal_frames_for( + current, + history, + frame_offsets=HENET_LONG_FRAME_OFFSETS, + ) + return ( + _image_inputs_for( + short_frames, + output_height=HENET_INPUT_HEIGHT, + output_width=HENET_INPUT_WIDTH, + device=device, + ), + _image_inputs_for( + long_frames, + output_height=HENET_LONGTERM_INPUT_HEIGHT, + output_width=HENET_LONGTERM_INPUT_WIDTH, + device=device, + ), + ) + + +def henet_segmentation_from_result(result: Any) -> np.ndarray: + """Extract one official HENet sigmoid segmentation tensor.""" + if ( + not isinstance(result, Sequence) + or len(result) != 1 + or not isinstance(result[0], Mapping) + ): + raise ValueError("HENet result has an unexpected envelope") + segmentation = result[0].get("pts_seg") + if segmentation is None or not hasattr(segmentation, "detach"): + raise ValueError("HENet result is missing BEV segmentation") + values = np.asarray(segmentation.detach().cpu(), dtype=np.float32) + expected_shape = (len(HENET_SEGMENTATION_CLASS_NAMES), 200, 200) + if values.shape != expected_shape: + raise ValueError( + "HENet BEV segmentation has shape " + f"{list(values.shape)}, expected {list(expected_shape)}" + ) + if not np.isfinite(values).all() or np.any(values < 0.0) or np.any( + values > 1.0 + ): + raise ValueError("HENet BEV segmentation must be finite in [0,1]") + return values + + +def infer_henet_frame( + model: Any, + current: PackedHENetFrame, + history: Mapping[int, PackedHENetFrame], + *, + device: Any, +) -> np.ndarray: + """Run official HENet and return one ASOC `[8,450,300]` probability grid.""" + import torch + + short_inputs, long_inputs = henet_inputs_for( + current, + history, + device=device, + ) + placeholder = torch.zeros( + ( + len(HENET_SEGMENTATION_CLASS_NAMES), + 200, + 200, + ), + dtype=torch.float32, + device=device, + ) + with torch.no_grad(): + result = model.simple_test( + None, + [{}], + img=short_inputs, + img_lt=long_inputs, + gt_masks_bev=[placeholder], + ) + return adapt_henet_segmentation(henet_segmentation_from_result(result)) + + +def remember_history_frame( + history: MutableMapping[int, PackedHENetFrame], + frame: PackedHENetFrame, +) -> None: + """Retain only the 4 second history consumed by HENet long-term input.""" + stale_before = frame.frame_index + min(HENET_LONG_FRAME_OFFSETS) + for frame_index in list(history): + if frame_index < stale_before: + del history[frame_index] + history[frame.frame_index] = frame + + +def sha256_file(path: str | Path, chunk_size: int = 8 << 20) -> str: + digest = hashlib.sha256() + with Path(path).open("rb") as stream: + while chunk := stream.read(chunk_size): + digest.update(chunk) + return digest.hexdigest() + + +def load_official_henet( + *, + repository_path: str | Path, + checkpoint_path: str | Path, + checkpoint_sha256: str, + device: Any, +) -> Any: + """Load only the pinned HENet source revision and immutable checkpoint.""" + repository = Path(repository_path).resolve() + checkpoint = Path(checkpoint_path).resolve() + if sha256_file(checkpoint) != checkpoint_sha256: + raise ValueError("HENet weight digest does not match provenance") + git_head = repository / ".git" / "HEAD" + if not git_head.exists(): + raise ValueError("HENet repository has no revision metadata") + revision = subprocess.run( + ["git", "-C", str(repository), "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + if revision != HENET_REVISION: + raise ValueError("HENet repository revision is not pinned") + + sys.path.insert(0, str(repository)) + try: + from mmcv import Config + from mmcv.runner import load_checkpoint + from mmdet3d.models import build_model + + importlib.import_module("mmdet3d.models") + config = Config.fromfile( + str(repository / "configs" / "henet" / HENET_CONFIG_NAME) + ) + config.model.pretrained = None + config.model.train_cfg = None + model = build_model( + config.model, + test_cfg=config.get("test_cfg"), + ) + load_checkpoint( + model, + str(checkpoint), + map_location="cpu", + ) + # The Dashboard only publishes the native segmentation head. Avoid + # detector metadata requirements and unnecessary detector computation. + model.pts_bbox_head = None + model.to(device) + model.eval() + return model + finally: + if sys.path and sys.path[0] == str(repository): + sys.path.pop(0) From 9a06e007064863eaa217b11655a4c558d438d78e Mon Sep 17 00:00:00 2001 From: riita10069 Date: Sat, 22 Aug 2026 01:13:41 +0900 Subject: [PATCH 05/15] test(occupancy): validate HENet packed runtime contract Exercise camera ordering, temporal inputs, and segmentation result validation. Signed-off-by: riita10069 --- Model/tests/test_henet_runtime.py | 222 ++++++++++++++++++++++++++++++ 1 file changed, 222 insertions(+) create mode 100644 Model/tests/test_henet_runtime.py diff --git a/Model/tests/test_henet_runtime.py b/Model/tests/test_henet_runtime.py new file mode 100644 index 000000000..a8704ef1f --- /dev/null +++ b/Model/tests/test_henet_runtime.py @@ -0,0 +1,222 @@ +import io +import json +import tarfile + +import numpy as np +import pytest +import torch +from PIL import Image + +from data_processing.geospatial import encode_pose +from Platform.pipelines.henet_runtime import ( + HENET_INPUT_HEIGHT, + HENET_INPUT_WIDTH, + HENET_LONGTERM_INPUT_HEIGHT, + HENET_LONGTERM_INPUT_WIDTH, + PackedHENetFrame, + henet_inputs_for, + henet_segmentation_from_result, + iter_packed_henet_frames, + remember_history_frame, + temporal_frames_for, + temporal_substitution_count, +) + + +def _projection() -> np.ndarray: + matrix = np.repeat(np.eye(3, 4)[None], 6, axis=0) + matrix[:, 0, 0] = 800.0 + matrix[:, 1, 1] = 810.0 + matrix[:, 0, 2] = 128.0 + matrix[:, 1, 2] = 129.0 + return matrix + + +def _image_bytes(color=(10, 20, 30), size=(256, 256)) -> bytes: + output = io.BytesIO() + Image.new("RGB", size, color).save(output, format="PNG") + return output.getvalue() + + +def _frame( + frame_index: int, + *, + episode_id: str = "scene-a", + longitude: float = 139.0, +) -> PackedHENetFrame: + return PackedHENetFrame( + sample_uid=f"sample-{frame_index}", + episode_id=episode_id, + frame_index=frame_index, + timestamp_ns=frame_index * 100_000_000, + image_payloads=(_image_bytes(),) * 6, + projection_ref_to_camera=_projection(), + pose={ + "latitude_deg": 35.0, + "longitude_deg": longitude, + "heading_deg_cw_from_north": 0.0, + "timestamp_ns": frame_index * 100_000_000, + "gps_accuracy_m": float("nan"), + }, + ) + + +def _add_tar_member( + archive: tarfile.TarFile, + name: str, + payload: bytes, +) -> None: + info = tarfile.TarInfo(name) + info.size = len(payload) + archive.addfile(info, io.BytesIO(payload)) + + +def test_packed_tar_reader_reorders_kitscenes_cameras_for_henet(tmp_path): + tar_path = tmp_path / "train-000000.tar" + with tarfile.open(tar_path, "w") as archive: + uid = "sample-64" + members = { + **{ + f"cam_{camera}.jpg": _image_bytes((camera, 20, 30)) + for camera in range(6) + }, + "meta.json": json.dumps( + { + "sample_uid": uid, + "split_group_uid": "scene-a", + "frame_idx": 64, + } + ).encode(), + "calib.json": json.dumps( + { + "dataset": "kitscenes", + "projection": { + "type": "pinhole", + "matrix": _projection().tolist(), + "reference_frame": "top_lidar_flu", + }, + } + ).encode(), + "pose.npy": encode_pose( + { + "latitude_deg": 35.0, + "longitude_deg": 139.0, + "heading_deg_cw_from_north": 0.0, + "timestamp_ns": 6_400_000_000, + } + ), + } + for suffix, payload in members.items(): + _add_tar_member(archive, f"{uid}.{suffix}", payload) + + frames = list(iter_packed_henet_frames(tar_path)) + + assert len(frames) == 1 + assert [ + Image.open(io.BytesIO(payload)).getpixel((0, 0))[0] + for payload in frames[0].image_payloads + ] == [1, 0, 2, 4, 3, 5] + np.testing.assert_allclose( + frames[0].projection_ref_to_camera, + _projection()[[1, 0, 2, 4, 3, 5]], + ) + + +def test_temporal_selection_uses_exact_two_hz_history_and_key_fallback(): + current = _frame(100) + history = { + 60: _frame(60), + 90: _frame(90), + 95: _frame(95), + } + + selected = temporal_frames_for( + current, + history, + frame_offsets=(0, -5, -10, -15), + ) + + assert [ + frame.frame_index + for frame in selected.values() + ] == [100, 95, 90, 100] + assert temporal_substitution_count( + current, + history, + frame_offsets=(0, -5, -10, -15), + ) == 1 + + +def test_temporal_selection_rejects_cross_scene_history(): + with pytest.raises(ValueError, match="crossed"): + temporal_frames_for( + _frame(100), + {95: _frame(95, episode_id="scene-b")}, + frame_offsets=(0, -5), + ) + + +def test_input_builder_matches_official_short_and_longterm_shapes(): + short_inputs, long_inputs = henet_inputs_for( + _frame(100), + { + 95: _frame(95, longitude=139.0001), + 90: _frame(90, longitude=139.0002), + }, + device=torch.device("cpu"), + ) + + assert short_inputs[0].shape == ( + 1, + 18, + 3, + HENET_INPUT_HEIGHT, + HENET_INPUT_WIDTH, + ) + assert long_inputs[0].shape == ( + 1, + 54, + 3, + HENET_LONGTERM_INPUT_HEIGHT, + HENET_LONGTERM_INPUT_WIDTH, + ) + assert short_inputs[1].shape == (1, 18, 4, 4) + assert long_inputs[1].shape == (1, 54, 4, 4) + assert short_inputs[3].shape == (1, 18, 3, 3) + assert long_inputs[5].shape == (1, 54, 3) + + +def test_result_extraction_preserves_official_probability_tensor(): + values = torch.full((3, 200, 200), 0.75) + + output = henet_segmentation_from_result([{"pts_seg": values}]) + + assert output.shape == (3, 200, 200) + assert output.dtype == np.float32 + assert output.max() == pytest.approx(0.75) + + +@pytest.mark.parametrize( + "result", + [ + [], + [{}], + [{"pts_seg": torch.zeros((3, 200, 199))}], + [{"pts_seg": torch.full((3, 200, 200), 1.1)}], + ], +) +def test_result_extraction_rejects_invalid_model_output(result): + with pytest.raises(ValueError): + henet_segmentation_from_result(result) + + +def test_history_cache_retains_the_longterm_window_only(): + history = { + 59: _frame(59), + 60: _frame(60), + 61: _frame(61), + } + + remember_history_frame(history, _frame(100)) + + assert set(history) == {60, 61, 100} From 8809d2a90546b40370467de723a14303ed9a30fc Mon Sep 17 00:00:00 2001 From: riita10069 Date: Sat, 22 Aug 2026 01:14:20 +0900 Subject: [PATCH 06/15] build(occupancy): reserve HENet Flyte image binding Keep HENet's research runtime isolated from training and BEVFormer images. Signed-off-by: riita10069 --- Platform/pipelines/workflows.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Platform/pipelines/workflows.py b/Platform/pipelines/workflows.py index 38fccea7a..937335add 100644 --- a/Platform/pipelines/workflows.py +++ b/Platform/pipelines/workflows.py @@ -68,6 +68,10 @@ "AUTO_E2E_BEVFORMER_V2_IMAGE", f"{ECR_PREFIX}/auto-e2e/bevformer-v2:latest", ) +HENET_IMAGE = _os.environ.get( + "AUTO_E2E_HENET_IMAGE", + f"{ECR_PREFIX}/auto-e2e/henet:latest", +) MLFLOW_URI = "http://mlflow.mlflow.svc.cluster.local:5000" DATASET_PACK_VERSION = "v2.2" From a971fcd5bb85485cdcf629032e1f874f20b8dd85 Mon Sep 17 00:00:00 2001 From: riita10069 Date: Sat, 22 Aug 2026 01:15:30 +0900 Subject: [PATCH 07/15] feat(occupancy): publish HENet semantic artifact sets Record every HENet inference input in the immutable Dashboard publication identity. Signed-off-by: riita10069 --- Platform/pipelines/workflows.py | 338 ++++++++++++++++++++++++++++++++ 1 file changed, 338 insertions(+) diff --git a/Platform/pipelines/workflows.py b/Platform/pipelines/workflows.py index 937335add..4df7c0d17 100644 --- a/Platform/pipelines/workflows.py +++ b/Platform/pipelines/workflows.py @@ -8611,6 +8611,344 @@ def precompute_bevformer_v2_occupancy_artifacts( ) +@task( + container_image=HENET_IMAGE, + requests=Resources(cpu="4", mem="40Gi", gpu="1"), + limits=Resources(cpu="4", mem="40Gi", gpu="1"), + pod_template=_large_shm_pod_template(), +) +def precompute_henet_occupancy_artifacts( + checkpoint: FlyteFile, + checkpoint_sha256: str, + shard_dirs: List[FlyteDirectory], + dataset: str, + dataset_version: str, + dataset_manifest_sha256: str, + artifacts_bucket: str, + publication_timestamp: str, + aws_region: str = "us-west-2", + repository_path: str = "/opt/HENet", +) -> SemanticOccupancyPrecomputeOutput: + """Publish official HENet BEV segmentation for the Occupancy Dashboard.""" + import hashlib + import json + import re + from pathlib import Path + + import boto3 + import numpy as np + import torch + + from Platform.pipelines.henet_occupancy import ( + HENET_ARTIFACT_KIND, + HENET_CODE_LICENSE_SPDX, + HENET_CONFIG_NAME, + HENET_HEAD_VERSION, + HENET_INPUT_HEIGHT, + HENET_INPUT_WIDTH, + HENET_LONGTERM_INPUT_HEIGHT, + HENET_LONGTERM_INPUT_WIDTH, + HENET_LONG_FRAME_OFFSETS, + HENET_REPOSITORY, + HENET_REVISION, + HENET_SHORT_FRAME_OFFSETS, + HENET_TRAINING_DATA_LICENSE_SPDX, + HENET_WEIGHT_LICENSE_SPDX, + HENET_WEIGHT_SOURCE_URL, + provenance, + ) + from Platform.pipelines.henet_runtime import ( + infer_henet_frame, + iter_packed_henet_frames, + load_official_henet, + remember_history_frame, + temporal_substitution_count, + ) + from Platform.pipelines.occupancy_store import ( + encode_occupancy_set_manifest, + occupancy_model_artifact_id, + occupancy_set_manifest, + occupancy_set_s3_key, + ) + from Platform.pipelines.overlay_tasks import _put_s3_immutable + from Platform.pipelines.semantic_occupancy import ( + SEMANTIC_OCCUPANCY_GEOMETRY_ID, + SEMANTIC_OCCUPANCY_SCHEMA, + SEMANTIC_OCCUPANCY_TAXONOMY_VERSION, + encode_semantic_occupancy, + quantize_semantic_occupancy, + semantic_occupancy_s3_key, + ) + + if dataset != "kitscenes": + raise ValueError("HENet occupancy supports only KITScenes") + if not re.fullmatch(r"v[1-9][0-9]*\.[0-9]+", dataset_version): + raise ValueError("dataset_version must match v.") + for name, value in ( + ("checkpoint_sha256", checkpoint_sha256), + ("dataset_manifest_sha256", dataset_manifest_sha256), + ): + if not re.fullmatch(r"[0-9a-f]{64}", value): + raise ValueError(f"{name} must be a lowercase SHA-256") + for name, value in ( + ("artifacts_bucket", artifacts_bucket), + ("publication_timestamp", publication_timestamp), + ("aws_region", aws_region), + ("repository_path", repository_path), + ): + if not value: + raise ValueError(f"{name} must not be empty") + if not shard_dirs: + raise ValueError("shard_dirs must not be empty") + + packed_shards = [] + seen_shards = set() + for shard_directory in shard_dirs: + local_directory = Path(shard_directory.download()) + packed_manifest_path = local_directory / "manifest.json" + if not packed_manifest_path.is_file(): + raise FileNotFoundError( + f"packed manifest missing: {local_directory}" + ) + packed_manifest = json.loads( + packed_manifest_path.read_text(encoding="utf-8") + ) + packed_dataset_version = packed_manifest.get( + "dataset_version", + packed_manifest.get("version"), + ) + if ( + packed_manifest.get("dataset") != dataset + or packed_dataset_version != dataset_version + or int(packed_manifest.get("num_views", 0)) != 6 + or not packed_manifest.get("has_gps", False) + ): + raise ValueError( + "packed KITScenes manifest differs from the HENet input " + f"contract: {packed_manifest_path}" + ) + for tar_path in sorted(local_directory.glob("*.tar")): + if tar_path.name in seen_shards: + raise ValueError( + "semantic occupancy shard names are not unique: " + f"{tar_path.name}" + ) + seen_shards.add(tar_path.name) + packed_shards.append(tar_path) + if not packed_shards: + raise ValueError("packed directories contain no tar shards") + packed_shards.sort(key=lambda path: path.name) + + model_source = { + "code_license_spdx": HENET_CODE_LICENSE_SPDX, + "config": HENET_CONFIG_NAME, + "license_spdx": HENET_WEIGHT_LICENSE_SPDX, + "repository": HENET_REPOSITORY, + "repository_revision": HENET_REVISION, + "training_data_license_spdx": HENET_TRAINING_DATA_LICENSE_SPDX, + "weight_sha256": checkpoint_sha256, + "weight_source_url": HENET_WEIGHT_SOURCE_URL, + } + input_contract = ( + "kitscenes-packed-256-square-six-camera-to-henet-short-" + "640x1152-long-256x704-v1" + ) + producer_config = { + "input_height": HENET_INPUT_HEIGHT, + "input_width": HENET_INPUT_WIDTH, + "longterm_input_height": HENET_LONGTERM_INPUT_HEIGHT, + "longterm_input_width": HENET_LONGTERM_INPUT_WIDTH, + "probability_encoding": "uint8-rint-gzip-level-6-v1", + "random_seed": 0, + "short_temporal_frame_offsets": list(HENET_SHORT_FRAME_OFFSETS), + "longterm_temporal_frame_offsets": list(HENET_LONG_FRAME_OFFSETS), + "temporal_boundary_policy": "repeat-current-frame-v1", + } + model_artifact_id = occupancy_model_artifact_id( + artifact_kind=HENET_ARTIFACT_KIND, + artifact_schema=SEMANTIC_OCCUPANCY_SCHEMA, + geometry_id=SEMANTIC_OCCUPANCY_GEOMETRY_ID, + head_version=HENET_HEAD_VERSION, + input_contract=input_contract, + model_source=model_source, + producer_config=producer_config, + taxonomy_version=SEMANTIC_OCCUPANCY_TAXONOMY_VERSION, + ) + manifest_key = occupancy_set_s3_key( + dataset, + dataset_version, + model_artifact_id, + dataset_manifest_sha256, + ) + + np.random.seed(0) + torch.manual_seed(0) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(0) + torch.backends.cudnn.benchmark = False + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + checkpoint_path = str(checkpoint.download()) + model = load_official_henet( + repository_path=repository_path, + checkpoint_path=checkpoint_path, + checkpoint_sha256=checkpoint_sha256, + device=device, + ) + + s3 = boto3.client("s3", region_name=aws_region) + entries = [] + total_samples = 0 + substituted_history_frames = 0 + substituted_history_slots = 0 + history = {} + active_episode = None + last_frame_index = None + for tar_path in packed_shards: + sample_uids = [] + probabilities = [] + for frame in iter_packed_henet_frames(tar_path): + if active_episode != frame.episode_id: + history.clear() + active_episode = frame.episode_id + last_frame_index = None + if ( + last_frame_index is not None + and frame.frame_index <= last_frame_index + ): + raise ValueError( + "packed KITScenes frames are not strictly increasing " + f"within episode {frame.episode_id!r}" + ) + missing_slots = temporal_substitution_count( + frame, + history, + frame_offsets=HENET_SHORT_FRAME_OFFSETS, + ) + temporal_substitution_count( + frame, + history, + frame_offsets=HENET_LONG_FRAME_OFFSETS, + ) + if missing_slots: + substituted_history_frames += 1 + substituted_history_slots += missing_slots + probability = infer_henet_frame( + model, + frame, + history, + device=device, + ) + probabilities.append(quantize_semantic_occupancy(probability)) + sample_uids.append(frame.sample_uid) + remember_history_frame(history, frame) + last_frame_index = frame.frame_index + if not sample_uids: + raise ValueError(f"packed shard is empty: {tar_path}") + payload = encode_semantic_occupancy( + sample_uids, + np.stack(probabilities), + ) + payload_sha256 = hashlib.sha256(payload).hexdigest() + key = semantic_occupancy_s3_key( + model_artifact_id, + dataset_manifest_sha256, + dataset, + tar_path.name, + head_version=HENET_HEAD_VERSION, + ) + _put_s3_immutable( + s3, + bucket=artifacts_bucket, + key=key, + payload=payload, + metadata={ + "artifact-kind": HENET_ARTIFACT_KIND, + "dataset-manifest-sha256": dataset_manifest_sha256, + "geometry-id": SEMANTIC_OCCUPANCY_GEOMETRY_ID, + "head-version": HENET_HEAD_VERSION, + "model-artifact-id": model_artifact_id, + "payload-sha256": payload_sha256, + "sample-count": str(len(sample_uids)), + "schema": SEMANTIC_OCCUPANCY_SCHEMA, + "taxonomy-version": ( + SEMANTIC_OCCUPANCY_TAXONOMY_VERSION + ), + "weight-sha256": checkpoint_sha256, + }, + content_type="application/vnd.auto-e2e.semantic-occupancy", + content_encoding="gzip", + ) + entries.append({ + "byte_size": len(payload), + "sample_count": len(sample_uids), + "s3_key": key, + "sha256": payload_sha256, + "shard": tar_path.name, + "teacher_present": False, + }) + total_samples += len(sample_uids) + + metadata = provenance(checkpoint_sha256) + limitations = list(metadata["limitations"]) + limitations.append( + "The official weight is supplied at execution time and is not " + "redistributed in the AutoE2E container image." + ) + if substituted_history_frames: + limitations.append( + f"{substituted_history_frames} of {total_samples} frames " + f"substituted {substituted_history_slots} unavailable HENet " + "history slots with the current frame at scene or packed-sequence " + "boundaries." + ) + manifest = occupancy_set_manifest( + artifact_kind=HENET_ARTIFACT_KIND, + artifact_schema=SEMANTIC_OCCUPANCY_SCHEMA, + created_at=publication_timestamp, + dataset=dataset, + dataset_version=dataset_version, + dataset_manifest_sha256=dataset_manifest_sha256, + display_name="HENet BEV segmentation", + geometry_id=SEMANTIC_OCCUPANCY_GEOMETRY_ID, + head_version=HENET_HEAD_VERSION, + input_contract=input_contract, + limitations=limitations, + model_artifact_id=model_artifact_id, + model_family="HENet", + model_source=model_source, + producer_config=producer_config, + shards=entries, + supported_classes=metadata["supported_semantic_classes"], + taxonomy_version=SEMANTIC_OCCUPANCY_TAXONOMY_VERSION, + teacher_available=False, + ) + manifest_payload, manifest_sha256 = encode_occupancy_set_manifest( + manifest + ) + _put_s3_immutable( + s3, + bucket=artifacts_bucket, + key=manifest_key, + payload=manifest_payload, + metadata={ + "artifact-kind": HENET_ARTIFACT_KIND, + "dataset-manifest-sha256": dataset_manifest_sha256, + "manifest-sha256": manifest_sha256, + "model-artifact-id": model_artifact_id, + "sample-count": str(total_samples), + "schema": manifest["schema_version"], + "weight-sha256": checkpoint_sha256, + }, + content_type="application/json", + ) + return SemanticOccupancyPrecomputeOutput( + manifest_key=manifest_key, + manifest_sha256=manifest_sha256, + checkpoint_sha256=checkpoint_sha256, + shard_count=len(entries), + sample_count=total_samples, + ) + + # ============================================================ # Task: Offline RL # ============================================================ From b14a74308c1ebda86d2ca3755d2733a911829db9 Mon Sep 17 00:00:00 2001 From: riita10069 Date: Sat, 22 Aug 2026 01:16:14 +0900 Subject: [PATCH 08/15] feat(occupancy): expose HENet publication workflow Allow the controlled publication launcher to invoke immutable HENet inference. Signed-off-by: riita10069 --- Platform/pipelines/workflows.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/Platform/pipelines/workflows.py b/Platform/pipelines/workflows.py index 4df7c0d17..167a22805 100644 --- a/Platform/pipelines/workflows.py +++ b/Platform/pipelines/workflows.py @@ -11554,6 +11554,32 @@ def wf_precompute_bevformer_v2_occupancy( ) +@workflow +def wf_precompute_henet_occupancy( + checkpoint: FlyteFile, + checkpoint_sha256: str, + shard_dirs: List[FlyteDirectory], + dataset: str, + dataset_version: str, + dataset_manifest_sha256: str, + artifacts_bucket: str, + publication_timestamp: str, + aws_region: str = "us-west-2", +) -> SemanticOccupancyPrecomputeOutput: + """Publish official HENet segmentation for the Occupancy Dashboard.""" + return precompute_henet_occupancy_artifacts( + checkpoint=checkpoint, + checkpoint_sha256=checkpoint_sha256, + shard_dirs=shard_dirs, + dataset=dataset, + dataset_version=dataset_version, + dataset_manifest_sha256=dataset_manifest_sha256, + artifacts_bucket=artifacts_bucket, + publication_timestamp=publication_timestamp, + aws_region=aws_region, + ) + + @workflow def wf_evaluate_kitscenes_benchmark( checkpoint: FlyteFile, From bea988f9c34c8188dee1b090a0ad159b018a830b Mon Sep 17 00:00:00 2001 From: riita10069 Date: Sat, 22 Aug 2026 01:17:12 +0900 Subject: [PATCH 09/15] build(occupancy): package pinned HENet inference runtime Isolate HENet's CUDA and OpenMMLab dependencies for reproducible publication. Signed-off-by: riita10069 --- Platform/docker/henet/Dockerfile | 102 +++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 Platform/docker/henet/Dockerfile diff --git a/Platform/docker/henet/Dockerfile b/Platform/docker/henet/Dockerfile new file mode 100644 index 000000000..0d2dc74be --- /dev/null +++ b/Platform/docker/henet/Dockerfile @@ -0,0 +1,102 @@ +FROM nvidia/cuda:11.8.0-cudnn8-devel-ubuntu22.04 + +ARG DEBIAN_FRONTEND=noninteractive +ARG HENET_REVISION=29ca81dd109cabe0a0c53ee354c4a74ad1559740 +ARG TORCH_CUDA_ARCH_LIST="8.6+PTX" + +ENV CUDA_HOME=/usr/local/cuda +ENV FORCE_CUDA=1 +ENV PYTHONUNBUFFERED=1 +ENV TORCH_CUDA_ARCH_LIST=${TORCH_CUDA_ARCH_LIST} + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + git \ + libgl1 \ + libglib2.0-0 \ + ninja-build \ + python3.10 \ + python3.10-dev \ + python3-pip \ + && rm -rf /var/lib/apt/lists/* \ + && python3.10 -m pip install --no-cache-dir --upgrade \ + pip==23.3.2 \ + setuptools==69.0.3 \ + wheel==0.41.2 + +RUN python3.10 -m pip install --no-cache-dir \ + torch==2.0.1+cu118 \ + torchvision==0.15.2+cu118 \ + torchaudio==2.0.2+cu118 \ + -f https://download.pytorch.org/whl/torch_stable.html \ + && python3.10 -m pip install --no-cache-dir \ + mmcv-full==1.6.2 \ + -f https://download.openmmlab.com/mmcv/dist/cu118/torch2.0/index.html \ + && python3.10 -m pip install --no-cache-dir \ + mmdet==2.28.2 \ + mmsegmentation==0.30.0 + +RUN python3.10 -m pip install --no-cache-dir \ + aiobotocore==2.17.0 \ + boto3==1.35.93 \ + cloudpickle==2.2.1 \ + einops==0.7.0 \ + flytekit==1.13.15 \ + fsspec==2024.6.1 \ + fvcore==0.1.5.post20221221 \ + iopath==0.1.9 \ + kornia==0.7.1 \ + kubernetes==28.1.0 \ + lyft-dataset-sdk==0.0.8 \ + networkx==2.2 \ + numba==0.58.1 \ + nuscenes-devkit==1.1.11 \ + opencv-python==4.8.1.78 \ + pandas==1.4.4 \ + Pillow==10.1.0 \ + plyfile==1.0.2 \ + pyquaternion==0.9.9 \ + s3fs==2024.6.1 \ + scikit-image==0.19.3 \ + scipy==1.10.1 \ + shapely==1.8.5 \ + tensorboard==2.14.0 \ + timm==0.9.12 \ + trimesh==2.35.39 \ + yacs==0.1.9 + +RUN git clone https://github.com/VDIGPKU/HENet.git /opt/HENet \ + && git -C /opt/HENet checkout "${HENET_REVISION}" \ + && test "$(git -C /opt/HENet rev-parse HEAD)" = "${HENET_REVISION}" \ + && cd /opt/HENet/mmdet3d/ops/csrc \ + && python3.10 setup.py build_ext --inplace \ + && cd /opt/HENet/mmdet3d/ops/deformattn \ + && python3.10 setup.py build install \ + && cd /opt/HENet \ + && python3.10 setup.py develop \ + && cd /opt/HENet/detr2 \ + && python3.10 setup.py develop + +WORKDIR /workspace + +COPY Model/ /workspace/Model/ +COPY Platform/pipelines/ /workspace/Platform/pipelines/ + +ENV PYTHONPATH=/opt/HENet:/workspace/Model:/workspace + +RUN python3.10 -m pip check \ + && python3.10 -c \ + "import importlib, torch; \ +from mmcv import Config; \ +from mmdet3d.models import build_model; \ +from Platform.pipelines import henet_occupancy, henet_runtime, workflows; \ +assert henet_occupancy.HENET_REVISION; \ +assert henet_runtime.HENET_INPUT_WIDTH == 1152; \ +assert workflows.precompute_henet_occupancy_artifacts.name; \ +config=Config.fromfile('/opt/HENet/configs/henet/henet_det_bevseg.py'); \ +config.model.pretrained=None; \ +config.model.train_cfg=None; \ +build_model(config.model, test_cfg=config.get('test_cfg')); \ +assert torch.__version__.startswith('2.0.1')" From af339b10868cc530389324c5d182aaaca6cef7c5 Mon Sep 17 00:00:00 2001 From: riita10069 Date: Sat, 22 Aug 2026 01:17:42 +0900 Subject: [PATCH 10/15] build(platform): build and push HENet image Make the isolated HENet publication runtime available to Flyte by digest. Signed-off-by: riita10069 --- Platform/buildspec.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Platform/buildspec.yml b/Platform/buildspec.yml index e9d584fd6..84f5d9aee 100644 --- a/Platform/buildspec.yml +++ b/Platform/buildspec.yml @@ -20,6 +20,7 @@ phases: - (docker pull ${ECR_URL}/auto-e2e/training:latest || true) & - (docker pull ${ECR_URL}/auto-e2e/data-prep:latest || true) & - (docker pull ${ECR_URL}/auto-e2e/bevformer-v2:latest || true) & + - (docker pull ${ECR_URL}/auto-e2e/henet:latest || true) & - wait build: commands: @@ -57,6 +58,16 @@ phases: -t ${ECR_URL}/auto-e2e/bevformer-v2:${IMAGE_TAG} -f Platform/docker/bevformer-v2/Dockerfile . + # ── HENet ── independent CUDA 11.8 runtime for the official BEV + # segmentation checkpoint. Its OpenMMLab toolchain is incompatible with + # both the shared training image and the BEVFormer legacy runtime. + - echo "=== Building HENet occupancy image ===" + - DOCKER_BUILDKIT=1 docker build + --cache-from ${ECR_URL}/auto-e2e/henet:latest + --build-arg BUILDKIT_INLINE_CACHE=1 + -t ${ECR_URL}/auto-e2e/henet:${IMAGE_TAG} + -f Platform/docker/henet/Dockerfile . + # ── Push ALL images in parallel now that every image is built. ECR dedups # blobs so the training/eval/offline-rl push is paid once, and data-prep # ships in parallel — much faster than the old sequential push. @@ -72,6 +83,8 @@ phases: PID_D=$! docker push ${ECR_URL}/auto-e2e/bevformer-v2:${IMAGE_TAG} & PID_B=$! + docker push ${ECR_URL}/auto-e2e/henet:${IMAGE_TAG} & + PID_H=$! # wait -n was returning early on some CodeBuild agents; wait on each PID # explicitly so a push failure is surfaced. wait $PID_T && echo "training pushed" || exit 1 @@ -79,6 +92,7 @@ phases: wait $PID_O && echo "offline-rl pushed" || exit 1 wait $PID_D && echo "data-prep pushed" || exit 1 wait $PID_B && echo "bevformer-v2 pushed" || exit 1 + wait $PID_H && echo "henet pushed" || exit 1 cache: paths: From efeb70c1ba89d96dca0a0c217d119df3551ce5a7 Mon Sep 17 00:00:00 2001 From: riita10069 Date: Sat, 22 Aug 2026 01:18:11 +0900 Subject: [PATCH 11/15] infra(platform): provision HENet ECR repository Store the isolated HENet publication image under the managed lifecycle policy. Signed-off-by: riita10069 --- Platform/infra/modules/ecr/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Platform/infra/modules/ecr/main.tf b/Platform/infra/modules/ecr/main.tf index 5a45566ce..dd4914b83 100644 --- a/Platform/infra/modules/ecr/main.tf +++ b/Platform/infra/modules/ecr/main.tf @@ -1,7 +1,7 @@ variable "environment" { type = string } locals { - repositories = ["auto-e2e/training", "auto-e2e/data-prep", "auto-e2e/eval", "auto-e2e/offline-rl", "auto-e2e/bevformer-v2"] + repositories = ["auto-e2e/training", "auto-e2e/data-prep", "auto-e2e/eval", "auto-e2e/offline-rl", "auto-e2e/bevformer-v2", "auto-e2e/henet"] } resource "aws_ecr_repository" "this" { From 57ef791d396b94fbe7333b25080c76fa8e9b4408 Mon Sep 17 00:00:00 2001 From: riita10069 Date: Sat, 22 Aug 2026 01:20:21 +0900 Subject: [PATCH 12/15] build(occupancy): gate HENet publication by license Signed-off-by: riita10069 --- Platform/buildspec-publish-occupancy.yml | 59 ++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 4 deletions(-) diff --git a/Platform/buildspec-publish-occupancy.yml b/Platform/buildspec-publish-occupancy.yml index 8872d5af2..83b7cd18b 100644 --- a/Platform/buildspec-publish-occupancy.yml +++ b/Platform/buildspec-publish-occupancy.yml @@ -1,11 +1,13 @@ version: 0.2 -# Publish the pinned native and BEVFormer V2 occupancy sets for KITScenes v3.3. +# Publish pinned occupancy sets for KITScenes v3.3. # # Required start-build overrides: # IMAGE_TAG, PUBLICATION_TIMESTAMP, REPOSITORY_REVISION # Optional: -# BEVFORMER_CHECKPOINT_URI and per-image tags +# BEVFORMER_CHECKPOINT_URI and per-image tags. HENet publication additionally +# requires PUBLISH_HENET=true, HENET_CHECKPOINT_URI, +# HENET_CHECKPOINT_SHA256, and HENET_LICENSE_AUTHORIZATION=author-approved. env: shell: bash @@ -19,8 +21,13 @@ env: EVAL_IMAGE_TAG: "" OFFLINE_RL_IMAGE_TAG: "" BEVFORMER_V2_IMAGE_TAG: "" + HENET_IMAGE_TAG: "" AUTOE2E_CHECKPOINT_URI: "" BEVFORMER_CHECKPOINT_URI: "" + HENET_CHECKPOINT_URI: "" + HENET_CHECKPOINT_SHA256: "" + HENET_LICENSE_AUTHORIZATION: "" + PUBLISH_HENET: "false" PUBLISHED_SHARDS_URI: "" ARTIFACTS_BUCKET: "" DATASET: kitscenes @@ -42,6 +49,7 @@ phases: - test "${DATASET}" = "kitscenes" - test "${DATASET_VERSION}" = "v3.3" - test "${SCORE_THRESHOLD}" = "0.2" + - test "${PUBLISH_HENET}" = "true" -o "${PUBLISH_HENET}" = "false" - test "${DATASET_MANIFEST_SHA256}" = "31faf5d2ceef17522f1c79e6ab31558a2aceb6f75043a130d1f6b7da3ce6211d" - test "${PUBLICATION_TIMESTAMP}" != "" - test "${REPOSITORY_REVISION}" != "" @@ -61,7 +69,13 @@ phases: if [ -z "${ARTIFACTS_BUCKET}" ]; then ARTIFACTS_BUCKET="auto-e2e-platform-artifacts-${ACCOUNT_ID}" fi + if [ "${PUBLISH_HENET}" = "true" ]; then + test -n "${HENET_CHECKPOINT_URI}" + [[ "${HENET_CHECKPOINT_SHA256}" =~ ^[0-9a-f]{64}$ ]] + test "${HENET_LICENSE_AUTHORIZATION}" = "author-approved" + fi export AUTOE2E_CHECKPOINT_URI BEVFORMER_CHECKPOINT_URI + export HENET_CHECKPOINT_URI HENET_CHECKPOINT_SHA256 export PUBLISHED_SHARDS_URI ARTIFACTS_BUCKET - | verify_s3_sha256() { @@ -94,12 +108,19 @@ phases: "${PUBLISHED_SHARDS_URI%/}/manifest.json" \ "${DATASET_MANIFEST_SHA256}" \ /tmp/kitscenes-v3.3-manifest.json + if [ "${PUBLISH_HENET}" = "true" ]; then + verify_s3_sha256 \ + "${HENET_CHECKPOINT_URI}" \ + "${HENET_CHECKPOINT_SHA256}" \ + /tmp/henet-checkpoint.pth + fi - | TRAINING_IMAGE_TAG="${TRAINING_IMAGE_TAG:-${IMAGE_TAG}}" DATA_PREP_IMAGE_TAG="${DATA_PREP_IMAGE_TAG:-${IMAGE_TAG}}" EVAL_IMAGE_TAG="${EVAL_IMAGE_TAG:-${IMAGE_TAG}}" OFFLINE_RL_IMAGE_TAG="${OFFLINE_RL_IMAGE_TAG:-${IMAGE_TAG}}" BEVFORMER_V2_IMAGE_TAG="${BEVFORMER_V2_IMAGE_TAG:-${IMAGE_TAG}}" + HENET_IMAGE_TAG="${HENET_IMAGE_TAG:-${IMAGE_TAG}}" image_ref() { REPOSITORY="$1" TAG="$2" @@ -129,9 +150,15 @@ phases: AUTO_E2E_BEVFORMER_V2_IMAGE="$( image_ref bevformer-v2 "${BEVFORMER_V2_IMAGE_TAG}" )" || exit 1 + if [ "${PUBLISH_HENET}" = "true" ]; then + AUTO_E2E_HENET_IMAGE="$( + image_ref henet "${HENET_IMAGE_TAG}" + )" || exit 1 + fi export AUTO_E2E_TRAINING_IMAGE AUTO_E2E_EVAL_IMAGE export AUTO_E2E_OFFLINE_RL_IMAGE AUTO_E2E_DATA_PREP_IMAGE export AUTO_E2E_BEVFORMER_V2_IMAGE + export AUTO_E2E_HENET_IMAGE export ECR_PREFIX="${ECR_URL}" export PYTHONPATH="${CODEBUILD_SRC_DIR}/Model:${CODEBUILD_SRC_DIR}:${PYTHONPATH:-}" - | @@ -174,10 +201,24 @@ phases: "checkpoint": os.environ["BEVFORMER_CHECKPOINT_URI"], "score_threshold": float(os.environ["SCORE_THRESHOLD"]), } - for path, inputs in ( + inputs_by_path = ( ("/tmp/native-occupancy-inputs.json", native), ("/tmp/bevformer-v2-occupancy-inputs.json", bevformer), - ): + ) + if os.environ["PUBLISH_HENET"] == "true": + inputs_by_path += ( + ( + "/tmp/henet-occupancy-inputs.json", + { + **common, + "checkpoint": os.environ["HENET_CHECKPOINT_URI"], + "checkpoint_sha256": os.environ[ + "HENET_CHECKPOINT_SHA256" + ], + }, + ), + ) + for path, inputs in inputs_by_path: with open(path, "w", encoding="utf-8") as stream: json.dump(inputs, stream, sort_keys=True) PY @@ -197,3 +238,13 @@ phases: Platform/pipelines/workflows.py \ wf_precompute_bevformer_v2_occupancy \ --inputs-file /tmp/bevformer-v2-occupancy-inputs.json + - | + if [ "${PUBLISH_HENET}" = "true" ]; then + pyflyte --config /tmp/flyte.yaml run --remote \ + --project "${FLYTE_PROJECT}" \ + --domain "${FLYTE_DOMAIN}" \ + --image "${AUTO_E2E_HENET_IMAGE}" \ + Platform/pipelines/workflows.py \ + wf_precompute_henet_occupancy \ + --inputs-file /tmp/henet-occupancy-inputs.json + fi From 010d978b6e13703701deb17c1118905ae57525c4 Mon Sep 17 00:00:00 2001 From: riita10069 Date: Sat, 22 Aug 2026 02:02:45 +0900 Subject: [PATCH 13/15] fix(occupancy): own the ASOC raster geometry Signed-off-by: riita10069 --- Platform/pipelines/semantic_occupancy.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/Platform/pipelines/semantic_occupancy.py b/Platform/pipelines/semantic_occupancy.py index 9336086db..f4982f4a4 100644 --- a/Platform/pipelines/semantic_occupancy.py +++ b/Platform/pipelines/semantic_occupancy.py @@ -15,15 +15,31 @@ import numpy as np import torch -from navigation.geometry import AUTOE2E_NAVIGATION_GEOMETRY +from navigation.geometry import NavigationRasterGeometry SEMANTIC_OCCUPANCY_SCHEMA = "v1" SEMANTIC_OCCUPANCY_FORMAT_VERSION = 1 SEMANTIC_OCCUPANCY_MAGIC = b"ASOC" SEMANTIC_OCCUPANCY_TAXONOMY_VERSION = "autoe2e-bev-semantic-v1" -SEMANTIC_OCCUPANCY_GEOMETRY_ID = ( - AUTOE2E_NAVIGATION_GEOMETRY.geometry_id +SEMANTIC_OCCUPANCY_GEOMETRY = NavigationRasterGeometry( + geometry_id="autoe2e-bev-450x300-0p4m-v1", + height_px=450, + width_px=300, + meters_per_pixel=0.4, + x_min_m=-60.0, + x_max_m=120.0, + y_min_m=-60.0, + y_max_m=60.0, + ego_anchor_row=299.5, + ego_anchor_col=149.5, + matching_pc_range=(-60.0, -60.0, -5.0, 120.0, 60.0, 3.0), + matching_bev_h=450, + matching_bev_w=300, + route_corridor_width_m=3.5, + destination_marker_radius_m=2.0, + route_rear_clip_m=10.0, ) +SEMANTIC_OCCUPANCY_GEOMETRY_ID = SEMANTIC_OCCUPANCY_GEOMETRY.geometry_id SEMANTIC_OCCUPANCY_HEAD_VERSION = "bev-segmentation-head-v1" SEMANTIC_OCCUPANCY_CLASS_NAMES = ( "drivable_area", From b598cac831909efb8c5a67bd2297ab12c3f49e8a Mon Sep 17 00:00:00 2001 From: riita10069 Date: Sat, 22 Aug 2026 02:03:03 +0900 Subject: [PATCH 14/15] fix(occupancy): share ASOC geometry with BEVFormer Signed-off-by: riita10069 --- Platform/pipelines/bevformer_v2_occupancy.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Platform/pipelines/bevformer_v2_occupancy.py b/Platform/pipelines/bevformer_v2_occupancy.py index 097180792..dfb02d531 100644 --- a/Platform/pipelines/bevformer_v2_occupancy.py +++ b/Platform/pipelines/bevformer_v2_occupancy.py @@ -13,9 +13,9 @@ import numpy as np -from navigation.geometry import ( - AUTOE2E_NAVIGATION_GEOMETRY, - NavigationRasterGeometry, +from navigation.geometry import NavigationRasterGeometry +from Platform.pipelines.semantic_occupancy import ( + SEMANTIC_OCCUPANCY_GEOMETRY, ) BEVFORMER_V2_REPOSITORY = "https://github.com/fundamentalvision/BEVFormer" @@ -252,7 +252,7 @@ def scale_packed_projection( def rasterize_detection_boxes( detections: Sequence[DetectionBox], *, - geometry: NavigationRasterGeometry = AUTOE2E_NAVIGATION_GEOMETRY, + geometry: NavigationRasterGeometry = SEMANTIC_OCCUPANCY_GEOMETRY, score_threshold: float = 0.2, max_detections: int = 300, ) -> np.ndarray: From b94fcea47e55f661a6adbb2603c79fe5bbc48bb2 Mon Sep 17 00:00:00 2001 From: riita10069 Date: Sat, 22 Aug 2026 02:03:21 +0900 Subject: [PATCH 15/15] fix(occupancy): share ASOC geometry with HENet Signed-off-by: riita10069 --- Platform/pipelines/henet_occupancy.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Platform/pipelines/henet_occupancy.py b/Platform/pipelines/henet_occupancy.py index 0c38e96b7..ea9057b05 100644 --- a/Platform/pipelines/henet_occupancy.py +++ b/Platform/pipelines/henet_occupancy.py @@ -12,9 +12,9 @@ import numpy as np -from navigation.geometry import ( - AUTOE2E_NAVIGATION_GEOMETRY, - NavigationRasterGeometry, +from navigation.geometry import NavigationRasterGeometry +from Platform.pipelines.semantic_occupancy import ( + SEMANTIC_OCCUPANCY_GEOMETRY, ) HENET_REPOSITORY = "https://github.com/VDIGPKU/HENet" @@ -199,7 +199,7 @@ def _sample_henet_probability( def adapt_henet_segmentation( probability: np.ndarray, *, - geometry: NavigationRasterGeometry = AUTOE2E_NAVIGATION_GEOMETRY, + geometry: NavigationRasterGeometry = SEMANTIC_OCCUPANCY_GEOMETRY, ) -> np.ndarray: """Map official HENet probabilities to the fixed ASOC taxonomy and grid.