diff --git a/.config.example b/.config.example index 961af7e..3ab5728 100644 --- a/.config.example +++ b/.config.example @@ -1 +1,3 @@ -PROJECTS=project_1,project_2,project_3 \ No newline at end of file +PROJECTS=project_1,project_2,project_3 +OLLAMA_API_URL="http://localhost:11434" +OLLAMA_MODEL="llama3.3" diff --git a/.gitignore b/.gitignore index 2ec2572..9bb3b4f 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ -.config \ No newline at end of file +.config +*/.ruff_cache/* +*/.venv/* +*/__pycache__/* \ No newline at end of file diff --git a/README.md b/README.md index 5dfd0c8..cf2d5b1 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,22 @@ Note: For AWS helpers, the AWS CLI is required for these commands to work. - Use force=true to skip confirmation prompt - Example: `tn heroku-delete-pipeline my-project true` +### Ollama as a local LLM +Use `tn update-config` to set a basic config the default for the local ollama is http://localhost:11434. You should be setting + +``` +OLLAMA_API_URL="http://localhost:11434" +OLLAMA_MODEL="llama3.3" +``` + +- `tn ollama-serve`: serve a local llm server this is needed to run any commands +- `tn ollama-pull`: When working locally if you want to try a model you have to pull it +- `tn ollama-run`: Run an interactive chat with the local ollama server +- `tn ollama-codegen message`: Use the code gen specific cli (WIP) + optional args: `--api_url=""` use a custom url other than the default (eg ngrok) + `--model=""` use a custom model the default is qwen2.5-coder:7b + + ## Contributing New Commands - aka "Recipes" It's easy to contribute new commands to tn-cli, simply add some commands to the `justfile` and open a Pull Request. diff --git a/justfile b/justfile index bbfff6d..a9d3ca5 100644 --- a/justfile +++ b/justfile @@ -1,6 +1,6 @@ [private] default: - just -f ~/.tn/cli/justfile --list + just -f ~/.tn/cli/justfile --list --unsorted [group('general')] os-info: @@ -10,7 +10,6 @@ os-info: [group('general')] install-uv: curl -LsSf https://astral.sh/uv/install.sh | sh - # # Re-clone and reinstall tn-cli # @@ -23,6 +22,50 @@ update: git clone git@github.com:thinknimble/tn-cli.git ~/.tn/cli fi +# +# Set up the config file +# +[group('config')] +init-config: + #!/usr/bin/env bash + ls -a ~/.tn + if [ -d ~/.tn/.config ]; then + echo "Config file already exists." + else + if [ -f ~/.tn/cli/.config.example ]; then + echo "Source config file exists: ~/.tn/cli/.config.example" + mkdir -p ~/.tn + cp ~/.tn/cli/.config.example ~/.tn/.config + if [ $? -eq 0 ]; then + echo "Config file copied successfully." + else + echo "Failed to copy config file." + fi + else + echo "Source config file does not exist: ~/.tn/cli/.config.example" + fi + fi +# +# Update the config +# + +[group('config')] +update-config var_name var_value: + #!/usr/bin/env bash + if [ ! -d ~/.tn/.config ]; then + echo "No config file found. Run 'tn config-init' first." + else + # read the file, if the variable exists update it otherwise add it + if grep -q "^$var_name=" ~/.tn/.config; then + sed -i "s/^$var_name=.*/$var_name=$var_value/" ~/.tn/.config + echo "Updated $var_name in config file." + else + echo "$var_name=$var_value" >> ~/.tn/.config + echo "Added $var_name to config file." + fi + fi + + # # Bootstrap new projects # @@ -93,6 +136,963 @@ aws-enable-bedrock project_name profile='default' region='us-east-1' model='*': aws cloudformation create-stack --stack-name {{project_name}}-bedrock-stack --template-url 'https://tn-s3-cloud-formation.s3.amazonaws.com/bedrock-user-permissions.yaml' --region {{region}} --parameters ParameterKey=ProjectName,ParameterValue={{project_name}} ParameterKey=AllowedModels,ParameterValue={{model}} --capabilities CAPABILITY_NAMED_IAM --profile={{profile}} # +# AWS Terraform Recipes +# +# Extracted from tn-spa-bootstrapper template scripts. +# Each recipe accepts parameters instead of relying on cookiecutter template variables. +# + +# Idempotent shared VPC creation with tagged resources +[group('aws-terraform')] +aws-setup-vpc vpc_name='tn-shared' environment='dev' profile='default' region='us-east-1': + #!/usr/bin/env bash + set -e + + VPC_NAME="{{vpc_name}}" + ENVIRONMENT="{{environment}}" + AWS_PROFILE="{{profile}}" + AWS_REGION="{{region}}" + + PROFILE_FLAG="" + if [[ "$AWS_PROFILE" != "default" ]]; then + PROFILE_FLAG="--profile $AWS_PROFILE" + fi + REGION_FLAG="--region $AWS_REGION" + + TAG_PREFIX="${VPC_NAME}-${ENVIRONMENT}" + + echo "Shared VPC Setup" + echo "================" + echo " VPC Name: $TAG_PREFIX" + echo " AWS Profile: $AWS_PROFILE" + echo " AWS Region: $AWS_REGION" + echo "" + + # Check AWS CLI + if ! command -v aws &> /dev/null; then + echo "Error: AWS CLI not found. Please install AWS CLI first." + exit 1 + fi + + if ! aws sts get-caller-identity $PROFILE_FLAG &> /dev/null; then + echo "Error: AWS CLI not configured for profile '$AWS_PROFILE'. Run 'aws configure' first." + exit 1 + fi + + # 1. Create or find VPC + echo "Checking for existing VPC..." + EXISTING_VPC=$(aws ec2 describe-vpcs \ + --filters "Name=tag:Name,Values=${TAG_PREFIX}" \ + $PROFILE_FLAG $REGION_FLAG \ + --query 'Vpcs[0].VpcId' --output text 2>/dev/null || echo "None") + + if [[ "$EXISTING_VPC" != "None" && -n "$EXISTING_VPC" ]]; then + VPC_ID="$EXISTING_VPC" + echo "VPC already exists: $VPC_ID" + else + echo "Creating VPC..." + VPC_ID=$(aws ec2 create-vpc \ + --cidr-block 10.0.0.0/16 \ + $PROFILE_FLAG $REGION_FLAG \ + --query 'Vpc.VpcId' --output text) + + aws ec2 modify-vpc-attribute --vpc-id "$VPC_ID" --enable-dns-support $PROFILE_FLAG $REGION_FLAG + aws ec2 modify-vpc-attribute --vpc-id "$VPC_ID" --enable-dns-hostnames $PROFILE_FLAG $REGION_FLAG + + aws ec2 create-tags --resources "$VPC_ID" \ + --tags Key=Name,Value="${TAG_PREFIX}" \ + $PROFILE_FLAG $REGION_FLAG + echo "VPC created: $VPC_ID" + fi + + # 2. Create or find Internet Gateway + echo "" + echo "Checking for Internet Gateway..." + EXISTING_IGW=$(aws ec2 describe-internet-gateways \ + --filters "Name=tag:Name,Values=${TAG_PREFIX}-igw" \ + $PROFILE_FLAG $REGION_FLAG \ + --query 'InternetGateways[0].InternetGatewayId' --output text 2>/dev/null || echo "None") + + if [[ "$EXISTING_IGW" != "None" && -n "$EXISTING_IGW" ]]; then + IGW_ID="$EXISTING_IGW" + echo "Internet Gateway already exists: $IGW_ID" + else + echo "Creating Internet Gateway..." + IGW_ID=$(aws ec2 create-internet-gateway \ + $PROFILE_FLAG $REGION_FLAG \ + --query 'InternetGateway.InternetGatewayId' --output text) + + aws ec2 attach-internet-gateway --internet-gateway-id "$IGW_ID" --vpc-id "$VPC_ID" \ + $PROFILE_FLAG $REGION_FLAG + + aws ec2 create-tags --resources "$IGW_ID" \ + --tags Key=Name,Value="${TAG_PREFIX}-igw" \ + $PROFILE_FLAG $REGION_FLAG + echo "Internet Gateway created and attached: $IGW_ID" + fi + + # 3. Create or find Route Table + echo "" + echo "Checking for Route Table..." + EXISTING_RT=$(aws ec2 describe-route-tables \ + --filters "Name=tag:Name,Values=${TAG_PREFIX}-rt" \ + $PROFILE_FLAG $REGION_FLAG \ + --query 'RouteTables[0].RouteTableId' --output text 2>/dev/null || echo "None") + + if [[ "$EXISTING_RT" != "None" && -n "$EXISTING_RT" ]]; then + RT_ID="$EXISTING_RT" + echo "Route Table already exists: $RT_ID" + else + echo "Creating Route Table..." + RT_ID=$(aws ec2 create-route-table --vpc-id "$VPC_ID" \ + $PROFILE_FLAG $REGION_FLAG \ + --query 'RouteTable.RouteTableId' --output text) + + aws ec2 create-route --route-table-id "$RT_ID" \ + --destination-cidr-block 0.0.0.0/0 \ + --gateway-id "$IGW_ID" \ + $PROFILE_FLAG $REGION_FLAG + + aws ec2 create-tags --resources "$RT_ID" \ + --tags Key=Name,Value="${TAG_PREFIX}-rt" \ + $PROFILE_FLAG $REGION_FLAG + echo "Route Table created with default route: $RT_ID" + fi + + # 4. Create subnets across 2 AZs + echo "" + echo "Checking for subnets..." + + # Get available AZs + AZS=($(aws ec2 describe-availability-zones \ + --filters "Name=state,Values=available" \ + $PROFILE_FLAG $REGION_FLAG \ + --query 'AvailabilityZones[0:2].ZoneName' --output text)) + + SUBNET_CIDRS=("10.0.1.0/24" "10.0.2.0/24") + + for i in 0 1; do + AZ="${AZS[$i]}" + CIDR="${SUBNET_CIDRS[$i]}" + SUBNET_NAME="${TAG_PREFIX}-subnet-${AZ}" + + EXISTING_SUBNET=$(aws ec2 describe-subnets \ + --filters "Name=tag:Name,Values=${SUBNET_NAME}" "Name=vpc-id,Values=${VPC_ID}" \ + $PROFILE_FLAG $REGION_FLAG \ + --query 'Subnets[0].SubnetId' --output text 2>/dev/null || echo "None") + + if [[ "$EXISTING_SUBNET" != "None" && -n "$EXISTING_SUBNET" ]]; then + echo "Subnet already exists in ${AZ}: $EXISTING_SUBNET" + else + echo "Creating subnet in ${AZ} (${CIDR})..." + SUBNET_ID=$(aws ec2 create-subnet \ + --vpc-id "$VPC_ID" \ + --cidr-block "$CIDR" \ + --availability-zone "$AZ" \ + $PROFILE_FLAG $REGION_FLAG \ + --query 'Subnet.SubnetId' --output text) + + aws ec2 modify-subnet-attribute --subnet-id "$SUBNET_ID" \ + --map-public-ip-on-launch $PROFILE_FLAG $REGION_FLAG + + aws ec2 associate-route-table --route-table-id "$RT_ID" --subnet-id "$SUBNET_ID" \ + $PROFILE_FLAG $REGION_FLAG > /dev/null + + aws ec2 create-tags --resources "$SUBNET_ID" \ + --tags Key=Name,Value="${SUBNET_NAME}" \ + $PROFILE_FLAG $REGION_FLAG + echo "Subnet created in ${AZ}: $SUBNET_ID" + fi + done + + echo "" + echo "VPC setup complete!" + echo " VPC: $VPC_ID (${TAG_PREFIX})" + echo " Internet Gateway: $IGW_ID (${TAG_PREFIX}-igw)" + echo " Route Table: $RT_ID (${TAG_PREFIX}-rt)" + echo " Subnets: 2 across ${AZS[0]} and ${AZS[1]}" + +# Create S3 bucket and DynamoDB table for Terraform remote state backend +[group('aws-terraform')] +aws-tf-setup-backend service profile='default': + #!/usr/bin/env bash + set -e + + SERVICE="{{service}}" + AWS_PROFILE="{{profile}}" + + PROFILE_FLAG="" + if [[ "$AWS_PROFILE" != "default" ]]; then + PROFILE_FLAG="--profile $AWS_PROFILE" + fi + + # Check AWS CLI + if ! command -v aws &> /dev/null; then + echo "Error: AWS CLI not found. Please install AWS CLI first." + exit 1 + fi + + if ! aws sts get-caller-identity $PROFILE_FLAG &> /dev/null; then + echo "Error: AWS CLI not configured for profile '$AWS_PROFILE'. Run 'aws configure --profile $AWS_PROFILE' first." + exit 1 + fi + + echo "AWS CLI configured for profile: $AWS_PROFILE" + + # Get AWS account info + AWS_ACCOUNT_ID=$(aws sts get-caller-identity $PROFILE_FLAG --query Account --output text) + AWS_REGION=$(aws configure get region $PROFILE_FLAG 2>/dev/null || echo "us-east-1") + + echo "" + echo "Terraform S3 Backend Setup" + echo "==========================" + echo " Service: $SERVICE" + echo " Profile: $AWS_PROFILE" + echo " Account ID: $AWS_ACCOUNT_ID" + echo " Region: $AWS_REGION" + + # Generate standard names + BUCKET_NAME="${AWS_ACCOUNT_ID}-${SERVICE}-terraform-state" + TABLE_NAME="${SERVICE}-terraform-state-lock" + + echo "" + echo "Resources to create:" + echo " S3 Bucket: $BUCKET_NAME" + echo " DynamoDB Table: $TABLE_NAME" + echo "" + + echo -n "Proceed? (y/N): " + read confirm + if [[ ! "$confirm" =~ ^[Yy]$ ]]; then + echo "Cancelled" + exit 0 + fi + + # Create S3 bucket (idempotent) + echo "" + echo "Creating S3 bucket: $BUCKET_NAME" + if aws s3api head-bucket --bucket "$BUCKET_NAME" --region "$AWS_REGION" $PROFILE_FLAG 2>/dev/null; then + echo "S3 bucket '$BUCKET_NAME' already exists" + else + if [[ "$AWS_REGION" == "us-east-1" ]]; then + aws s3api create-bucket --bucket "$BUCKET_NAME" --region "$AWS_REGION" $PROFILE_FLAG + else + aws s3api create-bucket --bucket "$BUCKET_NAME" --region "$AWS_REGION" \ + --create-bucket-configuration LocationConstraint="$AWS_REGION" $PROFILE_FLAG + fi + + aws s3api put-bucket-versioning --bucket "$BUCKET_NAME" \ + --versioning-configuration Status=Enabled $PROFILE_FLAG + + aws s3api put-bucket-encryption --bucket "$BUCKET_NAME" \ + --server-side-encryption-configuration '{ + "Rules": [{"ApplyServerSideEncryptionByDefault":{"SSEAlgorithm":"AES256"},"BucketKeyEnabled":true}] + }' $PROFILE_FLAG + + aws s3api put-public-access-block --bucket "$BUCKET_NAME" \ + --public-access-block-configuration \ + BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true \ + $PROFILE_FLAG + + echo "S3 bucket created with versioning, encryption, and public access blocked" + fi + + # Create DynamoDB table (idempotent) + echo "" + echo "Creating DynamoDB table: $TABLE_NAME" + if aws dynamodb describe-table --table-name "$TABLE_NAME" --region "$AWS_REGION" $PROFILE_FLAG 2>/dev/null; then + echo "DynamoDB table '$TABLE_NAME' already exists" + else + aws dynamodb create-table \ + --table-name "$TABLE_NAME" \ + --attribute-definitions AttributeName=LockID,AttributeType=S \ + --key-schema AttributeName=LockID,KeyType=HASH \ + --billing-mode PAY_PER_REQUEST \ + --region "$AWS_REGION" $PROFILE_FLAG + + echo "Waiting for DynamoDB table to be active..." + aws dynamodb wait table-exists --table-name "$TABLE_NAME" --region "$AWS_REGION" $PROFILE_FLAG + echo "DynamoDB table created" + fi + + echo "" + echo "Backend setup complete!" + echo " S3 Bucket: $BUCKET_NAME" + echo " DynamoDB Table: $TABLE_NAME" + echo " Region: $AWS_REGION" + echo "" + echo "Next: run 'tn aws-tf-init-backend $SERVICE ' to initialize Terraform" +# Initialize Terraform with the correct backend configuration for an environment +[group('aws-terraform')] +aws-tf-init-backend service environment='development' profile='default' region='us-east-1' force='false': + #!/usr/bin/env bash + set -e + + SERVICE="{{service}}" + ENVIRONMENT="{{environment}}" + AWS_PROFILE="{{profile}}" + AWS_REGION="{{region}}" + FORCE="{{force}}" + + PROFILE_FLAG="" + if [[ "$AWS_PROFILE" != "default" ]]; then + PROFILE_FLAG="--profile $AWS_PROFILE" + fi + + # Get AWS account ID + AWS_ACCOUNT_ID=$(aws sts get-caller-identity $PROFILE_FLAG --query Account --output text) + + # Derive backend resource names (must match setup-backend convention) + BUCKET="${AWS_ACCOUNT_ID}-${SERVICE}-terraform-state" + TABLE="${SERVICE}-terraform-state-lock" + STATE_KEY="${ENVIRONMENT}/terraform.tfstate" + + echo "Terraform Backend Initialization" + echo "==================================" + echo " Service: $SERVICE" + echo " Environment: $ENVIRONMENT" + echo " AWS Profile: $AWS_PROFILE" + echo " State Key: $STATE_KEY" + echo " S3 Bucket: $BUCKET" + echo " DynamoDB Table: $TABLE" + echo " Region: $AWS_REGION" + echo "" + + # Test AWS access + echo "Testing AWS access..." + AWS_IDENTITY=$(aws sts get-caller-identity $PROFILE_FLAG 2>/dev/null || echo "FAILED") + if [[ "$AWS_IDENTITY" == "FAILED" ]]; then + echo "Error: Failed to get AWS caller identity" + exit 1 + fi + echo "AWS Identity verified: $(echo "$AWS_IDENTITY" | jq -r '.Arn')" + + # Test DynamoDB table access + echo "Testing DynamoDB table access..." + if aws dynamodb describe-table --table-name "$TABLE" --region "$AWS_REGION" $PROFILE_FLAG &>/dev/null; then + echo "DynamoDB table accessible: $TABLE" + else + echo "Error: DynamoDB table not accessible: $TABLE" + echo "Tip: Run 'tn aws-tf-setup-backend $SERVICE' first" + exit 1 + fi + + # Generate backend.hcl for CI (idempotent — same values for all environments) + BACKEND_HCL="terraform/backend.hcl" + if [[ ! -f "$BACKEND_HCL" ]]; then + echo "Generating $BACKEND_HCL for CI..." + printf 'bucket = "%s"\nregion = "%s"\ndynamodb_table = "%s"\nencrypt = true\n' \ + "$BUCKET" "$AWS_REGION" "$TABLE" > "$BACKEND_HCL" + echo "Created $BACKEND_HCL — commit this file to your repo" + else + echo "Backend config already exists: $BACKEND_HCL" + fi + + # Build backend config args + BACKEND_ARGS="-backend-config=\"bucket=${BUCKET}\"" + BACKEND_ARGS+=" -backend-config=\"key=${STATE_KEY}\"" + BACKEND_ARGS+=" -backend-config=\"region=${AWS_REGION}\"" + BACKEND_ARGS+=" -backend-config=\"dynamodb_table=${TABLE}\"" + BACKEND_ARGS+=" -backend-config=\"encrypt=true\"" + + if [[ "$AWS_PROFILE" != "default" ]]; then + BACKEND_ARGS+=" -backend-config=\"profile=${AWS_PROFILE}\"" + fi + + if [[ "$FORCE" == "true" ]]; then + BACKEND_ARGS+=" -migrate-state" + fi + + echo "" + echo "Running terraform init..." + eval "terraform init ${BACKEND_ARGS}" + + echo "" + echo "Backend initialized!" + echo " State Location: s3://${BUCKET}/${STATE_KEY}" + echo " Lock Table: ${TABLE}" + echo " Environment: ${ENVIRONMENT}" +# Create GitHub Actions OIDC IAM role for a given org and environment. +# Can also be re-run to update an existing OIDC role with a new secrets bucket for a new project. +[group('aws-terraform')] +aws-setup-oidc service github_org secrets_bucket environment='development' profile='default' repo='': + #!/usr/bin/env bash + set -e + + SERVICE="{{service}}" + GITHUB_ORG="{{github_org}}" + ENVIRONMENT="{{environment}}" + SECRETS_BUCKET="{{secrets_bucket}}" + AWS_PROFILE="{{profile}}" + REPO="{{repo}}" + + if [[ -z "$SERVICE" ]]; then + echo "Error: service is required (e.g., 'my-project')" + exit 1 + fi + + if [[ -z "$GITHUB_ORG" ]]; then + echo "Error: github_org is required" + exit 1 + fi + + # Default repo to github_org/service if not provided + if [[ -z "$REPO" ]]; then + REPO="${GITHUB_ORG}/${SERVICE}" + fi + + # Set up AWS command helper + run_aws() { + if [[ "$AWS_PROFILE" != "default" && -n "$AWS_PROFILE" ]]; then + aws --profile "$AWS_PROFILE" "$@" + else + aws "$@" + fi + } + + ACCOUNT_ID=$(run_aws sts get-caller-identity --query Account --output text) + ROLE_NAME="github-actions-${SERVICE}-${ENVIRONMENT}" + + echo "GitHub Actions OIDC Setup" + echo "=========================" + echo " Service: $SERVICE" + echo " GitHub Org: $GITHUB_ORG" + echo " GitHub Repo: $REPO" + echo " Environment: $ENVIRONMENT" + echo " AWS Account: $ACCOUNT_ID" + echo " Role Name: $ROLE_NAME" + echo " AWS Profile: $AWS_PROFILE" + if [[ -n "$SECRETS_BUCKET" ]]; then + echo " Secrets Bucket: $SECRETS_BUCKET" + fi + echo "" + + # 1. Create OIDC Identity Provider (idempotent) + echo "Creating OIDC Identity Provider..." + if run_aws iam get-open-id-connect-provider \ + --open-id-connect-provider-arn "arn:aws:iam::${ACCOUNT_ID}:oidc-provider/token.actions.githubusercontent.com" &>/dev/null; then + echo "OIDC provider already exists" + else + run_aws iam create-open-id-connect-provider \ + --url https://token.actions.githubusercontent.com \ + --client-id-list sts.amazonaws.com \ + --thumbprint-list 1c58a3a8518e8759bf075b76b750d4f2df264fcd + echo "OIDC provider created" + fi + + # 2. Create or update IAM Role (idempotent) + echo "" + echo "Processing IAM Role: $ROLE_NAME" + # GitHub OIDC sub claims now include numeric IDs: org@ID/repo@ID + # Use wildcards after org and repo names to match both old and new formats + REPO_CONDITION="repo:${GITHUB_ORG}*/${SERVICE}*:*" + if run_aws iam get-role --role-name "$ROLE_NAME" &>/dev/null; then + echo "Role $ROLE_NAME already exists" + # Check if this specific repo already in trust policy + TRUST_POLICY=$(run_aws iam get-role --role-name "$ROLE_NAME" --query 'Role.AssumeRolePolicyDocument' --output json) + if echo "$TRUST_POLICY" | grep -q "repo:${REPO}:"; then + echo "Repo '$REPO' already has access in trust policy" + else + echo "Adding repo '$REPO' to trust policy..." + echo "$TRUST_POLICY" | jq --arg repo_cond "$REPO_CONDITION" ' + (.Statement[0].Condition.StringLike["token.actions.githubusercontent.com:sub"]) |= + if type == "string" then [., $repo_cond] + elif type == "array" then . + [$repo_cond] + else $repo_cond + end + ' > /tmp/oidc-trust-policy.json + run_aws iam update-assume-role-policy --role-name "$ROLE_NAME" \ + --policy-document file:///tmp/oidc-trust-policy.json + rm -f /tmp/oidc-trust-policy.json + echo "Trust policy updated — added repo: $REPO" + fi + else + jq -n --arg account "$ACCOUNT_ID" --arg repo_cond "$REPO_CONDITION" '{ + Version: "2012-10-17", + Statement: [{ + Effect: "Allow", + Principal: { Federated: "arn:aws:iam::\($account):oidc-provider/token.actions.githubusercontent.com" }, + Action: "sts:AssumeRoleWithWebIdentity", + Condition: { + StringEquals: { "token.actions.githubusercontent.com:aud": "sts.amazonaws.com" }, + StringLike: { "token.actions.githubusercontent.com:sub": $repo_cond } + } + }] + }' > /tmp/oidc-trust-policy.json + run_aws iam create-role --role-name "$ROLE_NAME" \ + --assume-role-policy-document file:///tmp/oidc-trust-policy.json + rm -f /tmp/oidc-trust-policy.json + echo "IAM role created: $ROLE_NAME (trust scoped to repo: $REPO)" + fi + + # 3. Create and attach deployment policy (idempotent) + echo "" + echo "Creating deployment policy..." + POLICY_NAME="${ROLE_NAME}-deployment-policy" + POLICY_ARN="arn:aws:iam::${ACCOUNT_ID}:policy/${POLICY_NAME}" + + jq -n '{ + Version: "2012-10-17", + Statement: [ + {Sid: "ECRFullAccess", Effect: "Allow", Action: ["ecr:*"], Resource: "*"}, + {Sid: "ECSFullAccess", Effect: "Allow", Action: ["ecs:*"], Resource: "*"}, + {Sid: "VPCAccess", Effect: "Allow", Action: ["ec2:Describe*","ec2:CreateVpc","ec2:DeleteVpc","ec2:ModifyVpcAttribute","ec2:CreateSubnet","ec2:DeleteSubnet","ec2:ModifySubnetAttribute","ec2:CreateInternetGateway","ec2:DeleteInternetGateway","ec2:AttachInternetGateway","ec2:DetachInternetGateway","ec2:CreateRouteTable","ec2:DeleteRouteTable","ec2:CreateRoute","ec2:DeleteRoute","ec2:AssociateRouteTable","ec2:DisassociateRouteTable","ec2:CreateSecurityGroup","ec2:DeleteSecurityGroup","ec2:AuthorizeSecurityGroupIngress","ec2:AuthorizeSecurityGroupEgress","ec2:RevokeSecurityGroupIngress","ec2:RevokeSecurityGroupEgress","ec2:CreateTags","ec2:DeleteTags"], Resource: "*"}, + {Sid: "RDSAccess", Effect: "Allow", Action: ["rds:*"], Resource: "*"}, + {Sid: "ACMAccess", Effect: "Allow", Action: ["acm:*"], Resource: "*"}, + {Sid: "IAMAccess", Effect: "Allow", Action: ["iam:*"], Resource: "*"}, + {Sid: "SecretsManagerAccess", Effect: "Allow", Action: ["secretsmanager:*"], Resource: "*"}, + {Sid: "CloudWatchLogsAccess", Effect: "Allow", Action: ["logs:*"], Resource: "*"}, + {Sid: "S3FullAccess", Effect: "Allow", Action: ["s3:*"], Resource: "*"}, + {Sid: "DynamoDBAccess", Effect: "Allow", Action: ["dynamodb:*"], Resource: "*"}, + {Sid: "ELBAccess", Effect: "Allow", Action: ["elasticloadbalancing:*"], Resource: "*"}, + {Sid: "ElastiCacheAccess", Effect: "Allow", Action: ["elasticache:*"], Resource: "*"}, + {Sid: "Route53Access", Effect: "Allow", Action: ["route53:*"], Resource: "*"}, + {Sid: "EventBridgeAccess", Effect: "Allow", Action: ["events:*"], Resource: "*"}, + {Sid: "STSAccess", Effect: "Allow", Action: ["sts:GetCallerIdentity"], Resource: "*"} + ] + }' > /tmp/oidc-deploy-policy.json + + # Delete existing policy if it exists (idempotent) + if run_aws iam get-policy --policy-arn "$POLICY_ARN" &>/dev/null; then + run_aws iam detach-role-policy --role-name "$ROLE_NAME" --policy-arn "$POLICY_ARN" 2>/dev/null || true + run_aws iam list-policy-versions --policy-arn "$POLICY_ARN" \ + --query 'Versions[?!IsDefaultVersion].[VersionId]' --output text | while read version; do + run_aws iam delete-policy-version --policy-arn "$POLICY_ARN" --version-id "$version" 2>/dev/null || true + done + run_aws iam delete-policy --policy-arn "$POLICY_ARN" 2>/dev/null || true + fi + + run_aws iam create-policy --policy-name "$POLICY_NAME" \ + --policy-document file:///tmp/oidc-deploy-policy.json + run_aws iam attach-role-policy --role-name "$ROLE_NAME" --policy-arn "$POLICY_ARN" + rm -f /tmp/oidc-deploy-policy.json + echo "Deployment policy attached" + + # 4. Create secrets policy (idempotent) + echo "" + echo "Creating S3 secrets policy..." + SECRETS_POLICY_NAME="${ROLE_NAME}-secrets-access" + SECRETS_POLICY_ARN="arn:aws:iam::${ACCOUNT_ID}:policy/${SECRETS_POLICY_NAME}" + + jq -n --arg bucket "$SECRETS_BUCKET" --arg env "$ENVIRONMENT" '{ + Version: "2012-10-17", + Statement: [ + {Sid: "SecretsS3Access", Effect: "Allow", Action: ["s3:GetObject","s3:PutObject","s3:DeleteObject","s3:GetObjectVersion"], Resource: ["arn:aws:s3:::\($bucket)/\($env)/*"]}, + {Sid: "AllowListBucketForEnv", Effect: "Allow", Action: "s3:ListBucket", Resource: "arn:aws:s3:::\($bucket)", Condition: {StringLike: {"s3:prefix": "\($env)/*"}}}, + {Sid: "AllowListBuckets", Effect: "Allow", Action: "s3:ListAllMyBuckets", Resource: "*"} + ] + }' > /tmp/oidc-secrets-policy.json + + if run_aws iam get-policy --policy-arn "$SECRETS_POLICY_ARN" &>/dev/null; then + run_aws iam detach-role-policy --role-name "$ROLE_NAME" --policy-arn "$SECRETS_POLICY_ARN" 2>/dev/null || true + run_aws iam list-policy-versions --policy-arn "$SECRETS_POLICY_ARN" \ + --query 'Versions[?!IsDefaultVersion].[VersionId]' --output text | while read version; do + run_aws iam delete-policy-version --policy-arn "$SECRETS_POLICY_ARN" --version-id "$version" 2>/dev/null || true + done + run_aws iam delete-policy --policy-arn "$SECRETS_POLICY_ARN" 2>/dev/null || true + fi + + run_aws iam create-policy --policy-name "$SECRETS_POLICY_NAME" \ + --policy-document file:///tmp/oidc-secrets-policy.json + run_aws iam attach-role-policy --role-name "$ROLE_NAME" --policy-arn "$SECRETS_POLICY_ARN" + rm -f /tmp/oidc-secrets-policy.json + echo "Secrets policy attached" + + # Summary + ROLE_ARN=$(run_aws iam get-role --role-name "$ROLE_NAME" --query Role.Arn --output text) + echo "" + echo "Setup complete!" + echo " Role ARN: $ROLE_ARN" + echo "" + echo "Copy this Role ARN into your .github/environments.json:" + echo " \"role_arn\": \"$ROLE_ARN\"" + echo "" + echo "Set it under the '$ENVIRONMENT' key alongside account_id, secrets_bucket, and region." +# Create S3 bucket for secrets storage with proper security +[group('aws-terraform')] +aws-setup-secrets service environment profile='default' region='us-east-1': + #!/usr/bin/env bash + set -e + + SERVICE="{{service}}" + ENVIRONMENT="{{environment}}" + AWS_PROFILE="{{profile}}" + AWS_REGION="{{region}}" + + PROFILE_FLAG="" + if [[ "$AWS_PROFILE" != "default" ]]; then + PROFILE_FLAG="--profile $AWS_PROFILE" + fi + + AWS_ACCOUNT_ID=$(aws sts get-caller-identity $PROFILE_FLAG --query Account --output text) + SECRETS_BUCKET="${SERVICE}-terraform-secrets" + + echo "S3 Secrets Bucket Setup" + echo "========================" + echo " Service: $SERVICE" + echo " Environment: $ENVIRONMENT" + echo " Account ID: $AWS_ACCOUNT_ID" + echo " Region: $AWS_REGION" + echo " Bucket: $SECRETS_BUCKET" + echo "" + + # Create bucket (idempotent) + if aws s3api head-bucket --bucket "$SECRETS_BUCKET" $PROFILE_FLAG 2>/dev/null; then + echo "Bucket '$SECRETS_BUCKET' already exists" + else + echo "Creating S3 bucket: $SECRETS_BUCKET" + if [[ "$AWS_REGION" == "us-east-1" ]]; then + aws s3api create-bucket --bucket "$SECRETS_BUCKET" $PROFILE_FLAG + else + aws s3api create-bucket --bucket "$SECRETS_BUCKET" --region "$AWS_REGION" \ + --create-bucket-configuration LocationConstraint="$AWS_REGION" $PROFILE_FLAG + fi + + # Enable versioning + aws s3api put-bucket-versioning --bucket "$SECRETS_BUCKET" \ + --versioning-configuration Status=Enabled $PROFILE_FLAG + + # Enable encryption + aws s3api put-bucket-encryption --bucket "$SECRETS_BUCKET" \ + --server-side-encryption-configuration '{ + "Rules": [{"ApplyServerSideEncryptionByDefault":{"SSEAlgorithm":"AES256"},"BucketKeyEnabled":true}] + }' $PROFILE_FLAG + + # Block public access + aws s3api put-public-access-block --bucket "$SECRETS_BUCKET" \ + --public-access-block-configuration \ + BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true \ + $PROFILE_FLAG + + echo "Bucket created with versioning, encryption, and public access blocked" + fi + + # Create/update bucket policy for OIDC role access (idempotent) + ROLE_NAME="github-actions-${SERVICE}-${ENVIRONMENT}" + ROLE_ARN="arn:aws:iam::${AWS_ACCOUNT_ID}:role/${ROLE_NAME}" + + echo "" + echo "Setting bucket policy for role: $ROLE_NAME" + + # Get existing policy or start fresh + EXISTING_POLICY=$(aws s3api get-bucket-policy --bucket "$SECRETS_BUCKET" --query 'Policy' --output text $PROFILE_FLAG 2>/dev/null || echo "") + + if [[ -n "$EXISTING_POLICY" ]]; then + BASE_POLICY="$EXISTING_POLICY" + else + BASE_POLICY='{"Version":"2012-10-17","Statement":[]}' + fi + + echo "$BASE_POLICY" | jq \ + --arg env "$ENVIRONMENT" \ + --arg role "$ROLE_ARN" \ + --arg bucket "$SECRETS_BUCKET" \ + --arg region "$AWS_REGION" ' + .Statement = [.Statement[] | select(.Principal.AWS != $role)] + + [ + { + Sid: "AllowAccess\($env)", + Effect: "Allow", + Principal: {AWS: $role}, + Action: ["s3:GetObject","s3:PutObject","s3:DeleteObject"], + Resource: ["arn:aws:s3:::\($bucket)/\($env)/*"], + Condition: {StringEquals: {"aws:RequestedRegion": $region}} + }, + { + Sid: "AllowList\($env)", + Effect: "Allow", + Principal: {AWS: $role}, + Action: ["s3:ListBucket"], + Resource: ["arn:aws:s3:::\($bucket)"], + Condition: {StringEquals: {"aws:RequestedRegion": $region}, StringLike: {"s3:prefix": "\($env)/*"}} + } + ] + ' > /tmp/secrets-bucket-policy.json + aws s3api put-bucket-policy --bucket "$SECRETS_BUCKET" \ + --policy file:///tmp/secrets-bucket-policy.json $PROFILE_FLAG + rm -f /tmp/secrets-bucket-policy.json + + echo "Bucket policy updated for environment: $ENVIRONMENT" + echo "" + echo "Secrets bucket setup complete!" + echo " Bucket: $SECRETS_BUCKET" + echo " Environment: $ENVIRONMENT" + echo " Role: $ROLE_NAME" +# Connect to a running ECS task via ECS Exec +[group('aws-terraform')] +aws-ecs-exec service environment='development' profile='default' region='us-east-1' command='bash': + #!/usr/bin/env bash + set -e + + SERVICE="{{service}}" + ENVIRONMENT="{{environment}}" + AWS_PROFILE="{{profile}}" + AWS_REGION="{{region}}" + COMMAND_CHOICE="{{command}}" + + PROFILE_FLAG="" + if [[ "$AWS_PROFILE" != "default" ]]; then + PROFILE_FLAG="--profile $AWS_PROFILE" + fi + REGION_FLAG="--region $AWS_REGION" + + CLUSTER_NAME="cluster-${SERVICE}-${ENVIRONMENT}" + + echo "ECS Exec - Connect to running tasks" + echo "=======================================" + echo " Service: $SERVICE" + echo " Environment: $ENVIRONMENT" + echo " Cluster: $CLUSTER_NAME" + echo " AWS Profile: $AWS_PROFILE" + echo " AWS Region: $AWS_REGION" + + # Check if cluster exists + if ! aws ecs describe-clusters --clusters "$CLUSTER_NAME" $PROFILE_FLAG $REGION_FLAG &>/dev/null; then + echo "Error: Cluster '$CLUSTER_NAME' not found" + exit 1 + fi + + # List available services + echo "" + echo "Available services:" + SERVICES=$(aws ecs list-services --cluster "$CLUSTER_NAME" $PROFILE_FLAG $REGION_FLAG --query 'serviceArns[*]' --output text) + + if [[ -z "$SERVICES" ]]; then + echo "Error: No services found in cluster '$CLUSTER_NAME'" + exit 1 + fi + + SERVICE_NAMES=() + i=1 + for service_arn in $SERVICES; do + service_name=$(basename "$service_arn") + SERVICE_NAMES+=("$service_name") + echo " $i) $service_name" + ((i++)) + done + + echo "" + read -p "Select service number (1): " SERVICE_CHOICE + SERVICE_CHOICE=${SERVICE_CHOICE:-1} + + if [[ "$SERVICE_CHOICE" -lt 1 || "$SERVICE_CHOICE" -gt ${#SERVICE_NAMES[@]} ]]; then + echo "Error: Invalid service selection" + exit 1 + fi + + SELECTED_SERVICE=${SERVICE_NAMES[$((SERVICE_CHOICE-1))]} + echo "Selected: $SELECTED_SERVICE" + + # Get running tasks + TASKS=$(aws ecs list-tasks --cluster "$CLUSTER_NAME" --service-name "$SELECTED_SERVICE" $PROFILE_FLAG $REGION_FLAG --desired-status RUNNING --query 'taskArns[*]' --output text) + + if [[ -z "$TASKS" ]]; then + echo "Error: No running tasks found for service '$SELECTED_SERVICE'" + exit 1 + fi + + TASK_ARNS=($TASKS) + if [[ ${#TASK_ARNS[@]} -gt 1 ]]; then + echo "" + echo "Multiple tasks found:" + for i in "${!TASK_ARNS[@]}"; do + task_id=$(basename "${TASK_ARNS[$i]}") + echo " $((i+1))) $task_id" + done + read -p "Select task number (1): " TASK_CHOICE + TASK_CHOICE=${TASK_CHOICE:-1} + SELECTED_TASK=${TASK_ARNS[$((TASK_CHOICE-1))]} + else + SELECTED_TASK=${TASK_ARNS[0]} + fi + + TASK_ID=$(basename "$SELECTED_TASK") + echo "Selected task: $TASK_ID" + + # Get container name + TASK_DEF=$(aws ecs describe-tasks --cluster "$CLUSTER_NAME" --tasks "$SELECTED_TASK" $PROFILE_FLAG $REGION_FLAG --query 'tasks[0].taskDefinitionArn' --output text) + CONTAINER_NAME=$(aws ecs describe-task-definition --task-definition "$TASK_DEF" $PROFILE_FLAG $REGION_FLAG --query 'taskDefinition.containerDefinitions[0].name' --output text) + echo "Container: $CONTAINER_NAME" + + # Parse command choice + case $COMMAND_CHOICE in + bash) COMMAND="/bin/bash" ;; + sh) COMMAND="/bin/sh" ;; + django) COMMAND="python manage.py shell" ;; + dbshell) COMMAND="python manage.py dbshell" ;; + *) COMMAND="$COMMAND_CHOICE" ;; + esac + + echo "" + echo "Connecting... (command: $COMMAND)" + echo "Type 'exit' to disconnect" + echo "====================" + + aws ecs execute-command \ + --cluster "$CLUSTER_NAME" \ + --task "$TASK_ID" \ + --container "$CONTAINER_NAME" \ + --interactive \ + --command "$COMMAND" \ + $PROFILE_FLAG $REGION_FLAG +# Stream CloudWatch logs from ECS services +[group('aws-terraform')] +aws-stream-logs service environment='development' profile='default' region='us-east-1' stream_type='a' filter='' duration='5m': + #!/usr/bin/env bash + set -e + + SERVICE="{{service}}" + ENVIRONMENT="{{environment}}" + AWS_PROFILE="{{profile}}" + AWS_REGION="{{region}}" + STREAM_TYPE="{{stream_type}}" + FILTER_PATTERN="{{filter}}" + START_TIME="{{duration}}" + + PROFILE_FLAG="" + if [[ "$AWS_PROFILE" != "default" ]]; then + PROFILE_FLAG="--profile $AWS_PROFILE" + fi + REGION_FLAG="--region $AWS_REGION" + + LOG_GROUP="/ecs/${SERVICE}/${ENVIRONMENT}" + + echo "ECS Logs Streaming" + echo "=============================================" + echo " Service: $SERVICE" + echo " Environment: $ENVIRONMENT" + echo " Log Group: $LOG_GROUP" + echo " AWS Profile: $AWS_PROFILE" + echo " AWS Region: $AWS_REGION" + + # Check if log group exists + if ! aws logs describe-log-groups --log-group-name-prefix "$LOG_GROUP" $PROFILE_FLAG $REGION_FLAG --query 'logGroups[?logGroupName==`'"$LOG_GROUP"'`]' --output text | grep -q "$LOG_GROUP"; then + echo "Error: Log group '$LOG_GROUP' not found" + echo "Tip: Make sure your service is deployed and running" + exit 1 + fi + + # Get available log streams + STREAMS=$(aws logs describe-log-streams \ + --log-group-name "$LOG_GROUP" \ + --order-by LastEventTime \ + --descending \ + --max-items 20 \ + $PROFILE_FLAG $REGION_FLAG \ + --query 'logStreams[*].logStreamName' \ + --output text) + + if [[ -z "$STREAMS" ]]; then + echo "Error: No log streams found in '$LOG_GROUP'" + exit 1 + fi + + # Categorize streams + SERVER_STREAMS=() + WORKER_STREAMS=() + OTHER_STREAMS=() + + i=1 + for stream in $STREAMS; do + if [[ "$stream" =~ server- ]]; then + SERVER_STREAMS+=("$stream") + echo " $i) [SERVER] $stream" + elif [[ "$stream" =~ worker- ]]; then + WORKER_STREAMS+=("$stream") + echo " $i) [WORKER] $stream" + else + OTHER_STREAMS+=("$stream") + echo " $i) [OTHER] $stream" + fi + ((i++)) + done + + ALL_STREAMS=("${SERVER_STREAMS[@]}" "${WORKER_STREAMS[@]}" "${OTHER_STREAMS[@]}") + + SELECTED_STREAMS=() + case $STREAM_TYPE in + a|A) SELECTED_STREAMS=("${SERVER_STREAMS[@]}"); echo "Streaming all server logs" ;; + w|W) SELECTED_STREAMS=("${WORKER_STREAMS[@]}"); echo "Streaming all worker logs" ;; + '*') SELECTED_STREAMS=("${ALL_STREAMS[@]}"); echo "Streaming all logs" ;; + *) + if [[ "$STREAM_TYPE" =~ ^[0-9]+$ ]] && [[ "$STREAM_TYPE" -ge 1 ]] && [[ "$STREAM_TYPE" -le ${#ALL_STREAMS[@]} ]]; then + SELECTED_STREAMS=("${ALL_STREAMS[$((STREAM_TYPE-1))]}") + echo "Streaming: ${ALL_STREAMS[$((STREAM_TYPE-1))]}" + else + echo "Error: Invalid stream selection" + exit 1 + fi + ;; + esac + + if [[ ${#SELECTED_STREAMS[@]} -eq 0 ]]; then + echo "Error: No streams selected" + exit 1 + fi + + # Validate duration format + if [[ ! "$START_TIME" =~ ^[0-9]+[mh]$ ]]; then + echo "Error: Invalid duration format. Use '30m' or '2h'" + exit 1 + fi + + # Build filter command + FILTER_CMD="aws logs filter-log-events --log-group-name \"$LOG_GROUP\" --start-time \$(date -v-${START_TIME} +%s)000 $PROFILE_FLAG $REGION_FLAG" + + if [[ -n "$FILTER_PATTERN" ]]; then + FILTER_CMD="$FILTER_CMD --filter-pattern \"$FILTER_PATTERN\"" + fi + + if [[ ${#SELECTED_STREAMS[@]} -lt ${#ALL_STREAMS[@]} ]]; then + STREAM_NAMES=$(IFS=' '; echo "${SELECTED_STREAMS[*]}") + FILTER_CMD="$FILTER_CMD --log-stream-names $STREAM_NAMES" + fi + + echo "" + echo "Log Group: $LOG_GROUP" + echo "Streams: ${#SELECTED_STREAMS[@]} selected" + echo "Time Range: Last $START_TIME" + if [[ -n "$FILTER_PATTERN" ]]; then + echo "Filter: $FILTER_PATTERN" + fi + echo "" + echo "Press Ctrl+C to stop streaming" + echo "====================" + + # Stream logs with continuous updates + LAST_SEEN="" + while true; do + CMD="$FILTER_CMD --output json" + if [[ -n "$LAST_SEEN" ]]; then + CMD="$CMD --next-token $LAST_SEEN" + fi + + RESPONSE=$(eval "$CMD" 2>/dev/null || echo '{"events":[],"nextToken":null}') + EVENTS=$(echo "$RESPONSE" | jq -c '.events[]?' 2>/dev/null) + NEXT_TOKEN=$(echo "$RESPONSE" | jq -r '.nextToken // empty' 2>/dev/null) + + if [[ -n "$EVENTS" ]]; then + while IFS= read -r event; do + if [[ -n "$event" ]]; then + timestamp=$(echo "$event" | jq -r '.timestamp // empty') + message=$(echo "$event" | jq -r '.message // empty') + stream=$(echo "$event" | jq -r '.logStreamName // empty') + if [[ -n "$timestamp" && -n "$message" ]]; then + formatted_time=$(date -r "$((timestamp/1000))" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || echo "$timestamp") + stream_short=$(basename "$stream") + echo "[$formatted_time] [$stream_short] $message" + fi + fi + done <<< "$EVENTS" + fi + + if [[ -n "$NEXT_TOKEN" && "$NEXT_TOKEN" != "null" ]]; then + LAST_SEEN="$NEXT_TOKEN" + fi + + sleep 2 + done +# # TN Models Helpers # # TODO: Support output, e.g. `-o somepath.json` @@ -205,8 +1205,6 @@ gh-all-prs: echo "" done - # Ollama CLI - # # Ollama CLI # @@ -231,16 +1229,23 @@ ollama-serve: ollama serve [group('ollama')] -ollama-gen: - ollama generate +ollama-pull model: + ollama pull {{model}} [group('ollama')] -ollama-codegen: - ollama - +ollama-run model: + ollama run {{model}} + + [group('ollama')] -ollama-customgen: - ollama +ollama-codegen message api_url='' model='qwen2.5-coder:7b': + #!/usr/bin/env bash + echo "Running Ollama codegen... using base model {{model}}" + echo "To clear the chat history, type 'clear chat history' and press enter." + CLEAN_API_URL=$(echo "{{api_url}}" | sed 's/^--api_url=//') + CLEAN_MODEL=$(echo "{{model}}" | sed 's/^--model=//') + + uvx uv run ./llm/ollama_client.py "{{message}}" --api_url="$CLEAN_API_URL" --model="$CLEAN_MODEL" # repo should be like: `owner/repo_name` [group('github')] diff --git a/llm/ollama_client.py b/llm/ollama_client.py new file mode 100644 index 0000000..0b9d8ec --- /dev/null +++ b/llm/ollama_client.py @@ -0,0 +1,124 @@ +# /// script +# requires-python = ">=3.12" +# dependencies = ["ollama", "pydantic==2.10.6", "pydantic_ai==0.0.43"] +# /// + +import asyncio +import argparse +from pydantic_ai.models.openai import OpenAIModel +from pydantic_ai import Agent +from pydantic_ai.messages import ( + FinalResultEvent, + FunctionToolCallEvent, + FunctionToolResultEvent, + PartDeltaEvent, + PartStartEvent, + TextPartDelta, + ToolCallPartDelta, +) +from pydantic_ai.tools import RunContext + +# Argument parsing +parser = argparse.ArgumentParser(description="Args for your call") +parser.add_argument("message", help="Message to process") +parser.add_argument("--api_url", type=str, help="Ollama API URL") +parser.add_argument("--model", type=str, default="qwen2.5-coder:7b", help="Ollama model") +args = parser.parse_args() + +model = None +api_url = None + + +# first try to read the config file, if it exists +def open_file_or_warn(filename): + try: + with open(filename, "r") as f: + return f.read() + except FileNotFoundError: + print(f"Warning: {filename} not found") + return "" + +config = open_file_or_warn("~/.tn/.config") + +for line in config.splitlines(): + if line.startswith("OLLAMA_API_URL"): + api_url = line.split("=")[1].strip() + elif line.startswith("OLLAMA_MODEL"): + model = line.split("=")[1].strip() + +# cli args always override config file +model = args.model if args.model else model if model else "llama3.3" +api_url = args.api_url if args.api_url else api_url if api_url else "http://localhost:11434" +if api_url.endswith("/"): + api_url = api_url[:-1] + +print(args.api_url, api_url, args.model, model) + + +OLLAMA_MODEL = OpenAIModel(model_name=model, base_url=api_url + "/v1") + +agent = Agent(model=OLLAMA_MODEL, system_prompt="You are a helpful assistant") + +output_messages = [] + +async def main(): + async with agent.iter(args.message) as run: + async for node in run: + if Agent.is_user_prompt_node(node): + # A user prompt node => The user has provided input + output_messages.append(f'=== UserPromptNode: {node.user_prompt} ===') + print(node.user_prompt) + print("\n") + elif Agent.is_model_request_node(node): + # A model request node => We can stream tokens from the model's request + output_messages.append( + '=== ModelRequestNode: streaming partial request tokens ===' + ) + async with node.stream(run.ctx) as request_stream: + async for event in request_stream: + if isinstance(event, PartStartEvent): + output_messages.append( + f'[Request] Starting part {event.index}: {event.part!r}' + ) + print(event.part.content, end='', flush=True) + + elif isinstance(event, PartDeltaEvent): + if isinstance(event.delta, TextPartDelta): + output_messages.append( + f'[Request] Part {event.index} text delta: {event.delta.content_delta!r}' + ) + print(event.delta.content_delta, end='', flush=True) + elif isinstance(event.delta, ToolCallPartDelta): + output_messages.append( + f'[Request] Part {event.index} args_delta={event.delta.args_delta}' + ) + + elif isinstance(event, FinalResultEvent): + output_messages.append( + f'[Result] The model produced a final result (tool_name={event.tool_name})' + ) + + elif Agent.is_call_tools_node(node): + # A handle-response node => The model returned some data, potentially calls a tool + output_messages.append( + '=== CallToolsNode: streaming partial response & tool usage ===' + ) + async with node.stream(run.ctx) as handle_stream: + async for event in handle_stream: + if isinstance(event, FunctionToolCallEvent): + output_messages.append( + f'[Tools] The LLM calls tool={event.part.tool_name!r} with args={event.part.args} (tool_call_id={event.part.tool_call_id!r})' + ) + elif isinstance(event, FunctionToolResultEvent): + output_messages.append( + f'[Tools] Tool call {event.tool_call_id!r} returned => {event.result.content}' + ) + elif Agent.is_end_node(node): + assert run.result.data == node.data.data + # Once an End node is reached, the agent run is complete + output_messages.append(f'=== Final Agent Output: {run.result.data} ===') + + +if __name__ == "__main__": + asyncio.run(main()) + # print(output_messages) \ No newline at end of file diff --git a/specs/aws-terraform-recipes/assertions/cli-recipe-group-listing.md b/specs/aws-terraform-recipes/assertions/cli-recipe-group-listing.md new file mode 100644 index 0000000..a9507a2 --- /dev/null +++ b/specs/aws-terraform-recipes/assertions/cli-recipe-group-listing.md @@ -0,0 +1,30 @@ +--- +id: cli-recipe-group-listing +parent: aws-terraform-recipes +created: 2026-07-20T23:00:00Z +priority: 1 +status: done +depends-on: cli-script-recipes +branch: feature/aws-pipeline + +--- + +# Recipe Group: All 7 Recipes Appear Under `[aws-terraform]` in `tn --list` + +## What Must Be True + +All seven AWS Terraform recipes (the VPC recipe plus the six extracted script recipes) are grouped under an `[aws-terraform]` section header in the justfile, and appear as a cohesive group in `tn --list` output. + +## Success Criteria + +- `tn --list` output includes an `[aws-terraform]` group header +- The following 7 recipes appear under that group: + - `aws-setup-vpc` + - `aws-ecs-exec` + - `aws-stream-logs` + - `aws-tf-setup-backend` + - `aws-tf-init-backend` + - `aws-setup-oidc` + - `aws-setup-secrets` +- No AWS Terraform recipes appear outside the `[aws-terraform]` group +- Group ordering is logical: VPC first, then backend setup, then OIDC/secrets, then operational tools (ecs-exec, stream-logs) diff --git a/specs/aws-terraform-recipes/assertions/cli-script-recipes.md b/specs/aws-terraform-recipes/assertions/cli-script-recipes.md new file mode 100644 index 0000000..b99a214 --- /dev/null +++ b/specs/aws-terraform-recipes/assertions/cli-script-recipes.md @@ -0,0 +1,37 @@ +--- +id: cli-script-recipes +parent: aws-terraform-recipes +created: 2026-07-20T23:00:00Z +priority: 1 +status: done +branch: feature/aws-pipeline +--- + +# Script Recipes: Six Extracted Operations Exist as tn-cli Recipes + +## What Must Be True + +The `tn-cli` justfile contains six recipes extracted from the bootstrapper template's scripts. Each recipe accepts parameters instead of relying on cookiecutter template variables, and contains no jinja template syntax. + +## Recipes + +| Recipe Name | Extracted From | +|---|---| +| `aws-ecs-exec` | `terraform/scripts/ecs-exec.sh` | +| `aws-stream-logs` | `terraform/scripts/stream-logs.sh` | +| `aws-tf-setup-backend` | `terraform/scripts/setup_backend.sh` | +| `aws-tf-init-backend` | `terraform/scripts/init_backend.sh` | +| `aws-setup-oidc` | `terraform/scripts/setup-github-oidc-role.sh` | +| `aws-setup-secrets` | `.github/scripts/setup-secrets-bucket.sh` | + +## Success Criteria + +- All 6 recipes exist in the justfile +- No `{% raw %}` / `{% endraw %}` jinja guards remain in any recipe code +- No cookiecutter template variables (`{{cookiecutter.*}}`) appear in any recipe +- Each recipe accepts project-specific values as parameters (e.g., project name, environment, region, profile) rather than hardcoded or template-derived values +- Each recipe is idempotent (safe to run multiple times without side effects) + +## Cross-Repo Dependency + +The bootstrapper repo's `template-extracted-scripts-removed` assertion depends on these recipes existing before the corresponding scripts are removed from the template. diff --git a/specs/aws-terraform-recipes/assertions/cli-vpc-recipe.md b/specs/aws-terraform-recipes/assertions/cli-vpc-recipe.md new file mode 100644 index 0000000..701fd3c --- /dev/null +++ b/specs/aws-terraform-recipes/assertions/cli-vpc-recipe.md @@ -0,0 +1,31 @@ +--- +id: cli-vpc-recipe +parent: aws-terraform-recipes +created: 2026-07-20T23:00:00Z +priority: 1 +status: done +branch: feature/aws-pipeline +--- + +# VPC Recipe: `aws-setup-vpc` Idempotently Creates a Tagged Shared VPC + +## What Must Be True + +The `tn-cli` justfile contains an `aws-setup-vpc` recipe that creates (or confirms existence of) a shared VPC with all required networking resources, tagged with a naming convention that Terraform data sources can filter on deterministically. + +## Success Criteria + +- Recipe exists in the justfile with parameters for `vpc_name` (default: `tn-shared`), `environment` (default: `dev`), `profile` (default: `default`), and `region` (default: `us-east-1`) +- Running the recipe twice produces no changes on the second run (idempotent) +- Created resources and their tags: + - VPC: Name tag `{vpc_name}-{environment}` (e.g., `tn-shared-dev`) + - Internet Gateway: Name tag `{vpc_name}-{environment}-igw` + - Route Table: Name tag `{vpc_name}-{environment}-rt` + - Subnets: at least 2 across different AZs, Name tags include `{vpc_name}-{environment}` +- No `{% raw %}` / `{% endraw %}` jinja guards in the recipe code +- No cookiecutter template variables (`{{cookiecutter.*}}`) in the recipe code +- Tag naming convention matches the filters used by `data "aws_vpc"`, `data "aws_internet_gateway"`, and `data "aws_route_table"` data sources in the bootstrapper's Terraform + +## Cross-Repo Dependency + +The bootstrapper repo's `tf-mandatory-vpc-data-sources` assertion depends on this recipe's tag naming convention. The data source filters in `main.tf` must match the tags this recipe creates. diff --git a/specs/aws-terraform-recipes/aws-terraform-recipes.md b/specs/aws-terraform-recipes/aws-terraform-recipes.md new file mode 100644 index 0000000..e24bc4a --- /dev/null +++ b/specs/aws-terraform-recipes/aws-terraform-recipes.md @@ -0,0 +1,38 @@ +--- +id: aws-terraform-recipes +created: 2026-07-20T23:00:00Z +priority: 1 +--- + +# AWS Terraform Recipes + +## Problem + +The tn-spa-bootstrapper template bundles one-time setup scripts (VPC creation, OIDC setup, backend initialization, secrets bucket, etc.) inside each generated project. These scripts contain no project-specific logic but are trapped inside the cookiecutter template, wrapped in `{% raw %}` / `{% endraw %}` jinja guards, and duplicated across every project that uses the template. + +Worse, the VPC creation logic lives inside per-project Terraform state, causing a race condition: multiple projects sharing a VPC can collide during concurrent `terraform apply` runs because each project believes it owns the VPC lifecycle. + +## Solution + +Extract all generic AWS infrastructure scripts into `tn-cli` as reusable justfile recipes under an `[aws-terraform]` group. Each recipe accepts parameters (project name, environment, region, profile) instead of relying on cookiecutter template variables. + +The key architectural change: VPC creation becomes a one-time `tn aws-setup-vpc` command run before any project deploys, rather than a conditional resource inside each project's Terraform. This eliminates the shared-state race condition entirely. + +## Recipes + +Seven recipes total: + +1. **`aws-setup-vpc`** -- Idempotent shared VPC creation with tagged resources +2. **`aws-ecs-exec`** -- Interactive ECS container exec session +3. **`aws-stream-logs`** -- CloudWatch log streaming +4. **`aws-tf-setup-backend`** -- S3 + DynamoDB backend creation +5. **`aws-tf-init-backend`** -- Terraform backend initialization +6. **`aws-setup-oidc`** -- GitHub OIDC role provisioning +7. **`aws-setup-secrets`** -- S3 secrets bucket creation + +## Constraints + +- Recipes must be pure shell -- no cookiecutter, no jinja +- All recipes must be idempotent (safe to run multiple times) +- Tag naming conventions must match the Terraform data source filters in the bootstrapper template +- Recipes appear under an `[aws-terraform]` group in `tn --list` output diff --git a/specs/oidc-per-project-isolation/assertions/deploy-policy-resource-scoped.md b/specs/oidc-per-project-isolation/assertions/deploy-policy-resource-scoped.md new file mode 100644 index 0000000..3675e93 --- /dev/null +++ b/specs/oidc-per-project-isolation/assertions/deploy-policy-resource-scoped.md @@ -0,0 +1,31 @@ +--- +id: deploy-policy-resource-scoped +parent: oidc-per-project-isolation +created: 2026-07-21T22:00:00Z +priority: 2 +status: draft + +--- + +# Deployment Policy Resources Are Scoped to Project + +## What Must Be True + +The OIDC role's deployment policy restricts resource access to the project's own resources using the `` naming convention, rather than granting `Resource: "*"` across the account. + +## Context + +Current policy grants `ecr:*`, `ecs:*`, `rds:*`, `s3:*`, `iam:*`, etc. on `Resource: "*"`. Any project's pipeline can access every other project's ECR repos, ECS services, RDS instances, and S3 buckets. + +This is the complex assertion — deferred until per-project roles and repo-scoped trust are in place. Resource scoping can be tightened via policy updates without migration. + +## Success Criteria + +- ECR access scoped to `arn:aws:ecr:::repository/-*` +- ECS access scoped to the project's cluster and services (`-*` prefix) +- RDS access scoped to `arn:aws:rds:::db:-*` and related sub-resources +- S3 access scoped to project-specific buckets (`-terraform-state`, `-terraform-secrets`) +- Secrets Manager access scoped to `arn:aws:secretsmanager:::secret:-*` +- CloudWatch Logs scoped to `arn:aws:logs:::log-group:/ecs/-*` +- IAM access scoped to roles/policies with `-*` prefix +- Shared read-only resources (VPC describe, ACM list, Route53 read) can remain `Resource: "*"` diff --git a/specs/oidc-per-project-isolation/assertions/oidc-role-per-project-per-env.md b/specs/oidc-per-project-isolation/assertions/oidc-role-per-project-per-env.md new file mode 100644 index 0000000..fb5b4c9 --- /dev/null +++ b/specs/oidc-per-project-isolation/assertions/oidc-role-per-project-per-env.md @@ -0,0 +1,26 @@ +--- +id: oidc-role-per-project-per-env +parent: oidc-per-project-isolation +created: 2026-07-21T22:00:00Z +priority: 2 +status: done +branch: feature/aws-pipeline +--- + +# OIDC Role Is Per-Project-Per-Environment + +## What Must Be True + +`aws-setup-oidc` accepts a required `service` parameter and creates a role named `github-actions--` (e.g., `github-actions-my-project-development`). Each project gets its own isolated role. + +## Context + +Currently the role is named `github-actions-` with no project scoping. The `service` parameter does not exist — all projects in an account share the same role. + +## Success Criteria + +- `aws-setup-oidc` signature includes a required `service` parameter (no default, errors if omitted) +- Role name follows pattern `github-actions--` +- Policy names follow pattern `github-actions---deployment-policy` and `github-actions---secrets-access` +- Command output prints the full role ARN for the user to copy into `environments.json` +- Existing roles without service prefix are not affected (command only manages roles matching the new naming pattern) diff --git a/specs/oidc-per-project-isolation/assertions/trust-policy-repo-scoped.md b/specs/oidc-per-project-isolation/assertions/trust-policy-repo-scoped.md new file mode 100644 index 0000000..cccc4e2 --- /dev/null +++ b/specs/oidc-per-project-isolation/assertions/trust-policy-repo-scoped.md @@ -0,0 +1,25 @@ +--- +id: trust-policy-repo-scoped +parent: oidc-per-project-isolation +created: 2026-07-21T22:00:00Z +priority: 2 +status: done +branch: feature/aws-pipeline +--- + +# Trust Policy Is Scoped to Specific GitHub Repo + +## What Must Be True + +The OIDC role's trust policy allows assumption only from the specific GitHub repository, not the entire GitHub organization. + +## Context + +Currently the trust policy condition uses `repo:/*:*` which allows any repo in the org to assume the role. For per-project isolation, only the project's repo should be able to assume its role. + +## Success Criteria + +- `aws-setup-oidc` accepts a `repo` parameter (or derives it from `service` with a convention like `/`) +- Trust policy `StringLike` condition uses `repo:/:*` instead of `repo:/*:*` +- When updating an existing role's trust policy, new repo conditions are appended without removing existing ones (idempotent, additive) +- Command output confirms which repo was added to the trust policy diff --git a/specs/oidc-per-project-isolation/oidc-per-project-isolation.md b/specs/oidc-per-project-isolation/oidc-per-project-isolation.md new file mode 100644 index 0000000..634f38e --- /dev/null +++ b/specs/oidc-per-project-isolation/oidc-per-project-isolation.md @@ -0,0 +1,27 @@ +--- +id: oidc-per-project-isolation +created: 2026-07-21T22:00:00Z +priority: 2 +--- + +# OIDC Per-Project Isolation + +## Problem + +`aws-setup-oidc` creates one IAM role per environment (`github-actions-development`) shared across all projects in an AWS account. The deployment policy grants `Resource: "*"` access to ECR, ECS, RDS, S3, IAM, and more. Any project's GitHub Actions pipeline can access every other project's resources in the same account. + +Each new project deployed on the shared role increases the migration cost when isolation is eventually needed (e.g., multi-customer dashboard). + +## Desired State + +Each project gets its own OIDC role (`github-actions--`) with a trust policy scoped to the specific GitHub repo. This prevents cross-project resource access and cross-repo role assumption. + +## Scope + +This spec covers the tn-cli justfile changes. The template-side changes (environments.json, post-gen instructions) are tracked in `tn-spa-bootstrapper` under the same spec ID. + +## Constraints + +- Must be backwards-compatible: existing shared roles should still work until migrated +- Role naming must follow `github-actions--` convention +- `` must match the `sanitized_tf_service_name` / `SERVICE_NAME` used elsewhere