diff --git a/.gitignore b/.gitignore index 3259eea0..6f3d6356 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ dist/ # Azure credentials file *_creds.json +*-key.json # Environment variables file .env diff --git a/Dockerfile b/Dockerfile index e6835b15..297f46a6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -76,16 +76,16 @@ RUN echo $TZ > /etc/timezone && \ # Install yq (architecture-aware) RUN ARCH=$(uname -m) && \ if [ "$ARCH" = "x86_64" ]; then ARCH="amd64"; elif [ "$ARCH" = "aarch64" ]; then ARCH="arm64"; fi && \ - curl -sL https://github.com/mikefarah/yq/releases/download/v4.47.2/yq_linux_${ARCH}.tar.gz | tar xz && \ + curl -sL https://github.com/mikefarah/yq/releases/download/v4.48.1/yq_linux_${ARCH}.tar.gz | tar xz && \ mv yq_linux_${ARCH} /usr/bin/yq && \ rm -rf /tmp/* # Install Google Cloud SDK (architecture-aware) RUN ARCH=$(uname -m) && \ if [ "$ARCH" = "x86_64" ]; then \ - curl -sSL "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-541.0.0-linux-x86_64.tar.gz" -o google-cloud-sdk.tar.gz; \ + curl -sSL "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-545.0.0-linux-x86_64.tar.gz" -o google-cloud-sdk.tar.gz; \ elif [ "$ARCH" = "aarch64" ]; then \ - curl -sSL "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-541.0.0-linux-arm.tar.gz" -o google-cloud-sdk.tar.gz; \ + curl -sSL "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-545.0.0-linux-arm.tar.gz" -o google-cloud-sdk.tar.gz; \ fi && \ tar -xzf google-cloud-sdk.tar.gz && \ ./google-cloud-sdk/install.sh -q && \ diff --git a/README.md b/README.md index d6e4a0c7..4c99d921 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,27 @@ docker exec my-secrets worker env get API_KEY See [Authorization Guide](docs/authorization.md) for supported providers and credential formats (JSON, Base64, File Path). +### 💡 Simplified Deployment + +For easier deployment with automatic credential detection, use the [`@udx/worker-deployment`](https://www.npmjs.com/package/@udx/worker-deployment) CLI: + +```bash +# Install +npm install -g @udx/worker-deployment + +# Generate config +worker-config + +# Run with automatic GCP authentication +worker-run +``` + +Features: +- ✅ Auto-detects GCP credentials (service account keys, impersonation, workload identity) +- ✅ Zero-config for default file names +- ✅ Secure read-only mounts +- ✅ Interactive debugging mode + ### Development Setup ```bash diff --git a/deploy-gcp.yml b/deploy-gcp.yml new file mode 100644 index 00000000..074f7c50 --- /dev/null +++ b/deploy-gcp.yml @@ -0,0 +1,14 @@ +# npm install -g @udx/worker-deployment +# worker-run --config=deploy-gcp.yml + +--- +kind: workerDeployConfig +version: udx.io/worker-v1/deploy +config: + # Docker image + image: "usabilitydynamics/udx-worker:latest" + + env: + ACTORS_CLEANUP: "false" + + command: "gcloud auth list" diff --git a/docs/auth/README.md b/docs/auth/README.md new file mode 100644 index 00000000..f29fdf4b --- /dev/null +++ b/docs/auth/README.md @@ -0,0 +1,34 @@ +# Provider Authentication Guides + +This directory contains detailed authentication documentation for each supported cloud provider. + +## Available Guides + +- **[GCP Authentication](gcp.md)** - Complete guide for Google Cloud Platform + - Service Account Keys + - Workload Identity Tokens + - Service Account Impersonation + - worker-deployment CLI integration + +- **[Azure Authentication](azure.md)** - Coming soon +- **[AWS Authentication](aws.md)** - Coming soon +- **[Bitwarden Authentication](bitwarden.md)** - Coming soon + +## Quick Links + +- [Main Authorization Guide](../authorization.md) - Overview and general credential formats +- [Worker Configuration](../config.md) - Worker configuration reference +- [CLI Documentation](../CLI.md) - Worker CLI commands + +## Contributing + +When adding a new provider authentication guide, please follow this structure: + +1. **Authentication Methods** - List all supported authentication methods +2. **JSON Format Examples** - Show credential structure +3. **Usage Examples** - Provide practical examples (env vars, file paths, base64) +4. **Features** - Highlight key features and capabilities +5. **Best Practices** - Security and operational recommendations +6. **CLI Integration** - If applicable, show worker-deployment CLI usage + +See [gcp.md](gcp.md) as a reference template. diff --git a/docs/auth/aws.md b/docs/auth/aws.md new file mode 100644 index 00000000..d5d7850a --- /dev/null +++ b/docs/auth/aws.md @@ -0,0 +1,14 @@ +# AWS Authentication + +> **Coming Soon**: Detailed AWS authentication documentation + +## Quick Reference + +**Environment Variable:** `AWS_CREDS` + +**Supported Formats:** +- JSON +- Base64-encoded JSON +- File path + +For now, see the [general authorization guide](../authorization.md) for credential format examples. diff --git a/docs/auth/azure.md b/docs/auth/azure.md new file mode 100644 index 00000000..5bd2c822 --- /dev/null +++ b/docs/auth/azure.md @@ -0,0 +1,24 @@ +# Azure Authentication + +> **Coming Soon**: Detailed Azure authentication documentation + +## Quick Reference + +**Environment Variable:** `AZURE_CREDS` + +**Supported Formats:** +- JSON +- Base64-encoded JSON +- File path + +**Example:** +```json +{ + "client_id": "CLIENT_ID", + "client_secret": "CLIENT_SECRET", + "tenant_id": "TENANT_ID", + "subscription_id": "SUBSCRIPTION_ID" +} +``` + +For now, see the [general authorization guide](../authorization.md) for credential format examples. diff --git a/docs/auth/bitwarden.md b/docs/auth/bitwarden.md new file mode 100644 index 00000000..f1d2a040 --- /dev/null +++ b/docs/auth/bitwarden.md @@ -0,0 +1,14 @@ +# Bitwarden Authentication + +> **Coming Soon**: Detailed Bitwarden authentication documentation + +## Quick Reference + +**Environment Variable:** `BITWARDEN_CREDS` + +**Supported Formats:** +- JSON +- Base64-encoded JSON +- File path + +For now, see the [general authorization guide](../authorization.md) for credential format examples. diff --git a/docs/auth/gcp.md b/docs/auth/gcp.md new file mode 100644 index 00000000..1713cbed --- /dev/null +++ b/docs/auth/gcp.md @@ -0,0 +1,189 @@ +# GCP Authentication + +Google Cloud Platform supports multiple authentication methods, each suited for different use cases. + +## Authentication Methods + +### 1. Service Account Key (Most Common) + +Service account keys work for both local development and CI/CD environments. + +**JSON Format:** +```json +{ + "type": "service_account", + "project_id": "my-project-id", + "private_key_id": "key-id", + "private_key": "-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----\n", + "client_email": "my-sa@my-project.iam.gserviceaccount.com", + "client_id": "123456789", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/..." +} +``` + +**Usage:** +```bash +# Via environment variable +export GCP_CREDS='{"type":"service_account",...}' + +# Via file path +export GCP_CREDS="/path/to/service-account-key.json" + +# Via base64 encoding (recommended for CI/CD) +export GCP_CREDS=$(cat service-account-key.json | base64) +``` + +**Features:** +- ✅ Automatic `private_key` normalization (handles escaped newlines) +- ✅ Sets both `GOOGLE_APPLICATION_CREDENTIALS` and `GCP_CREDS` +- ✅ Authenticates with `gcloud` CLI +- ✅ Sets project automatically from `project_id` field + +--- + +### 2. Workload Identity Token (GitHub Actions / CI/CD) + +Keyless authentication using OIDC tokens - no service account keys needed! + +**JSON Format:** +```json +{ + "type": "external_account", + "audience": "//iam.googleapis.com/projects/PROJECT_NUMBER/locations/global/workloadIdentityPools/POOL_ID/providers/PROVIDER_ID", + "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", + "token_url": "https://sts.googleapis.com/v1/token", + "service_account_impersonation_url": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/SA_EMAIL:generateAccessToken", + "credential_source": { + "file": "/path/to/token", + "format": { + "type": "text" + } + } +} +``` + +**GitHub Actions Example:** +```yaml +- uses: google-github-actions/auth@v3 + id: auth + with: + workload_identity_provider: ${{ secrets.WIF_PROVIDER }} + service_account: ${{ secrets.WIF_SERVICE_ACCOUNT }} + +- name: Run Worker + env: + GCP_CREDS: ${{ steps.auth.outputs.credentials_file_path }} + run: | + docker run -e GCP_CREDS usabilitydynamics/udx-worker:latest +``` + +**Features:** +- ✅ No long-lived credentials +- ✅ Automatic token refresh +- ✅ Works with Google Cloud client libraries +- ✅ Recommended for CI/CD pipelines + +--- + +### 3. Service Account Impersonation (Local Development) + +Use your personal gcloud credentials to impersonate a service account - no key files needed! + +**Setup:** +```bash +# 1. Authenticate with gcloud +gcloud auth login + +# 2. Set up Application Default Credentials (required for Terraform/SDKs) +gcloud auth application-default login + +# 3. Grant yourself impersonation permission +gcloud iam service-accounts add-iam-policy-binding \ + my-sa@my-project.iam.gserviceaccount.com \ + --member="user:$(gcloud config get-value account)" \ + --role="roles/iam.serviceAccountTokenCreator" \ + --project=MY_PROJECT +``` + +**Usage with worker-deployment CLI:** +```yaml +# deploy.yml +config: + service_account: + email: "my-sa@my-project.iam.gserviceaccount.com" + image: "usabilitydynamics/udx-worker:latest" + command: "worker run my-task" +``` + +```bash +# Run with automatic impersonation +worker-run --config=deploy.yml +``` + +**Manual Docker Usage:** +```bash +# Generate impersonation credentials +gcloud auth application-default print-access-token > /tmp/token.txt + +# Run container with impersonation +docker run \ + -e GOOGLE_APPLICATION_CREDENTIALS=/home/udx/adc.json \ + -e CLOUDSDK_AUTH_ACCESS_TOKEN=$(cat /tmp/token.txt) \ + -v ~/.config/gcloud/application_default_credentials.json:/home/udx/adc.json:ro \ + usabilitydynamics/udx-worker:latest +``` + +**Features:** +- ✅ No service account key files +- ✅ Uses your personal credentials +- ✅ Temporary access tokens +- ✅ Easy permission management +- ✅ Works with Terraform, gcloud, and SDKs + +> **Note**: Impersonation bypasses the `gcp_authenticate()` function by setting `GOOGLE_APPLICATION_CREDENTIALS` and `CLOUDSDK_AUTH_ACCESS_TOKEN` directly. + +--- + +## Authentication Priority + +When multiple credential sources are available, the worker uses this priority: + +1. **`GOOGLE_APPLICATION_CREDENTIALS`** - If already set, skip authentication (used for impersonation) +2. **`GCP_CREDS`** - Process through `gcp_authenticate()` function: + - Detect credential type (service account key vs. workload identity token) + - Normalize service account keys (fix escaped newlines in `private_key`) + - Set `GOOGLE_APPLICATION_CREDENTIALS` + - Authenticate with `gcloud auth login --cred-file` + +--- + +## Using worker-deployment CLI + +The [`@udx/worker-deployment`](https://www.npmjs.com/package/@udx/worker-deployment) CLI simplifies GCP authentication: + +**Installation:** +```bash +npm install -g @udx/worker-deployment +``` + +**Quick Start:** +```bash +# Generate config template +worker-config + +# Edit deploy.yml with your settings + +# Run with automatic credential detection +worker-run +``` + +**Features:** +- ✅ Automatic credential detection (service account keys, impersonation, workload identity) +- ✅ Zero-config for default file names (`gcp-key.json`, `gcp-credentials.json`) +- ✅ Secure read-only mounts +- ✅ Support for custom credential paths + +See the [worker-deployment README](https://github.com/udx/worker-deployment) for detailed examples. diff --git a/docs/authorization.md b/docs/authorization.md index 1c7ebc46..daf5233e 100644 --- a/docs/authorization.md +++ b/docs/authorization.md @@ -13,6 +13,8 @@ The UDX Worker supports multiple cloud providers and services through environmen | GCP | `GCP_CREDS` | Google Cloud Platform credentials | | Bitwarden | `BITWARDEN_CREDS` | Bitwarden secrets management credentials | +> **💡 Tip**: For simplified deployment with automatic credential detection, use the [`@udx/worker-deployment`](https://www.npmjs.com/package/@udx/worker-deployment) CLI tool. + ## Credential Formats Credentials can be provided in three formats: @@ -74,6 +76,15 @@ AZURE_CREDS="/path/to/azure_credentials.json" > **Note**: Always use absolute paths in production environments to avoid path resolution issues. +## Provider-Specific Authentication + +For detailed authentication guides for each provider, see: + +- **[GCP Authentication](auth/gcp.md)** - Service account keys, workload identity, impersonation +- **[Azure Authentication](auth/azure.md)** - Coming soon +- **[AWS Authentication](auth/aws.md)** - Coming soon +- **[Bitwarden Authentication](auth/bitwarden.md)** - Coming soon + ## Credential Management | Flag | Default | Description | diff --git a/lib/auth/gcp.sh b/lib/auth/gcp.sh index ed987150..b5791cfa 100644 --- a/lib/auth/gcp.sh +++ b/lib/auth/gcp.sh @@ -3,14 +3,18 @@ # shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 source "${WORKER_LIB_DIR}/utils.sh" -# Function to set ADC credentials +# GCP Authentication Module +# Supports: Service Account Keys, Workload Identity Tokens +# All methods use: gcloud auth login --cred-file="$GOOGLE_APPLICATION_CREDENTIALS" # -# Example usage of the function -# gcp_authenticate "/path/to/your/gcp_creds.json" -# gcp_authenticate "${GCP_CREDS}" +# Note: Impersonation is handled externally by setting GOOGLE_APPLICATION_CREDENTIALS +# and CLOUDSDK_AUTH_ACCESS_TOKEN directly, bypassing this module. # +# Example usage: +# gcp_authenticate "/path/to/gcp_creds.json" +# gcp_authenticate "${GCP_CREDS}" -# Function to set ADC credentials +# Function to authenticate with GCP gcp_authenticate() { local creds_json="$1" @@ -29,43 +33,52 @@ gcp_authenticate() { return 0 fi - # Extract necessary fields from the JSON credentials - local clientEmail privateKey projectId + local creds_file="$LOCAL_CREDS_DIR/gcp_creds.json" - clientEmail=$(echo "$creds_content" | jq -r '.client_email') - privateKey=$(echo "$creds_content" | jq -r '.private_key' | sed 's/- /-\n/g' | sed 's/ -/\n-/g') - projectId=$(echo "$creds_content" | jq -r '.project_id') - - if [[ -z "$clientEmail" || -z "$privateKey" || -z "$projectId" ]]; then - log_error "GCP Authentication" "Missing required GCP credentials." - return 1 - fi - - if [ -f "$GCP_CREDS" ]; then - # If GCP_CREDS is a file path and exists, use it directly - export GOOGLE_APPLICATION_CREDENTIALS="$GCP_CREDS" - else + # Check if this is a service account key that needs private_key normalization + if echo "$creds_content" | jq -e '.private_key' >/dev/null 2>&1; then + # Service account key - normalize private_key field + local clientEmail privateKey projectId + + clientEmail=$(echo "$creds_content" | jq -r '.client_email') + privateKey=$(echo "$creds_content" | jq -r '.private_key') + projectId=$(echo "$creds_content" | jq -r '.project_id') - # Adjust privateKey formatting - # Replace "\\n" with actual new line, handle BEGIN and END markers + # Normalize private_key: handle escaped newlines and spacing issues privateKey=$(echo "$privateKey" | sed 's/\\n/\n/g' | sed 's/- /\n-/g' | sed 's/ -/-\n/g') + # Create normalized JSON file jq -n --arg clientEmail "$clientEmail" --arg privateKey "$privateKey" --arg projectId "$projectId" \ - '{type: "service_account", client_email: $clientEmail, private_key: $privateKey, project_id: $projectId}' > "$LOCAL_CREDS_DIR/gcp_creds.json" - - export GOOGLE_APPLICATION_CREDENTIALS="$LOCAL_CREDS_DIR/gcp_creds.json" + '{type: "service_account", client_email: $clientEmail, private_key: $privateKey, project_id: $projectId}' > "$creds_file" + else + # Other credential types (workload identity, impersonation) - use as-is + echo "$creds_content" > "$creds_file" fi - # If GOOGLE_APPLICATION_CREDENTIALS is set, authorize environment with provided credentials - if [ -n "$GOOGLE_APPLICATION_CREDENTIALS" ]; then - log_info "GCP Authentication" "Authorizing environment with provided credentials." - gcloud auth login --cred-file="$GOOGLE_APPLICATION_CREDENTIALS" > /dev/null 2>&1 - fi + # Set GOOGLE_APPLICATION_CREDENTIALS for all methods + export GOOGLE_APPLICATION_CREDENTIALS="$creds_file" + + # Set GCP_CREDS for backward compatibility + export GCP_CREDS="$creds_file" - if ! gcloud config set project "$projectId" >/dev/null 2>&1; then - log_error "GCP Authentication" "Failed to set GCP project." + # Authenticate with gcloud (works for all credential types) + log_info "GCP Authentication" "Authenticating with gcloud..." + if ! gcloud auth login --cred-file="$GOOGLE_APPLICATION_CREDENTIALS" >/dev/null 2>&1; then + log_error "GCP Authentication" "Failed to authenticate with gcloud." return 1 fi - log_success "GCP Authentication" "GCP service account authenticated and project set." + # Extract and set project ID if available + local projectId + projectId=$(echo "$creds_content" | jq -r '.project_id // empty' 2>/dev/null) + + if [[ -n "$projectId" && "$projectId" != "null" ]]; then + if ! gcloud config set project "$projectId" >/dev/null 2>&1; then + log_error "GCP Authentication" "Failed to set GCP project: $projectId" + return 1 + fi + log_success "GCP Authentication" "Authenticated successfully. Project: $projectId" + else + log_success "GCP Authentication" "Authenticated successfully." + fi } \ No newline at end of file