From 7c282279dbb27ee38583dfd9b228369828072a3c Mon Sep 17 00:00:00 2001 From: Keys <70819367+keys-i@users.noreply.github.com> Date: Sat, 10 Aug 2024 10:12:00 +0530 Subject: [PATCH 1/6] add: automate code quality and security Start language-aware checks, CodeQL, dependency review, OSV, audits, and security linting. --- .github/workflows/checks.yml | 133 ++++++++++++++++++++++++++++++++++ .github/workflows/quality.yml | 123 +++++++++++++++++++++++++++++++ 2 files changed, 256 insertions(+) create mode 100644 .github/workflows/checks.yml create mode 100644 .github/workflows/quality.yml diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml new file mode 100644 index 0000000..52e9144 --- /dev/null +++ b/.github/workflows/checks.yml @@ -0,0 +1,133 @@ +name: Checks + +on: + merge_group: + pull_request: + branches: [main] + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: checks-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + shell: + name: Shell / ${{ matrix.name }} + runs-on: ${{ matrix.os }} + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + include: + - name: Linux + os: ubuntu-latest + language: sh + - name: macOS + os: macos-latest + language: zsh + + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Check Linux script + if: matrix.language == 'sh' + run: | + sh -n sys/linux/clean + shellcheck --shell=dash --severity=warning --exclude=SC1007 sys/linux/clean + sys/linux/clean --self-test + + - name: Check macOS script + if: matrix.language == 'zsh' + run: | + zsh -n sys/macos/clean + sys/macos/clean --self-test + + powershell: + name: PowerShell / Windows + runs-on: windows-latest + timeout-minutes: 10 + + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Install PSScriptAnalyzer + shell: pwsh + run: Install-Module PSScriptAnalyzer -RequiredVersion 1.25.0 -Scope CurrentUser -Force + + - name: Analyze script + shell: pwsh + run: | + $findings = Invoke-ScriptAnalyzer ` + -Path sys/windows/clean.ps1 ` + -Severity Warning, Error ` + -ExcludeRule PSReviewUnusedParameter, PSUseShouldProcessForStateChangingFunctions, PSUseSingularNouns + if ($findings) { + $findings | Format-Table -AutoSize + exit 1 + } + + - name: Test script + shell: pwsh + run: ./sys/windows/clean.ps1 -SelfTest + + python: + name: Python / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Install uv + if: hashFiles('dev/gui.py') != '' + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + version: "0.11.32" + python-version: "3.11" + + - name: Check formatting + if: hashFiles('dev/gui.py') != '' + run: uvx ruff@0.16.0 format --check --target-version=py311 dev/gui.py + + - name: Lint + if: hashFiles('dev/gui.py') != '' + run: uvx ruff@0.16.0 check --output-format=github --target-version=py311 dev/gui.py + + - name: Test dashboard + if: hashFiles('dev/gui.py') != '' + run: uv run dev/gui.py --self-test + + metadata: + name: Repository files + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Check Markdown and YAML formatting + run: npx --yes prettier@3.9.6 --check '**/*.md' '.github/**/*.{yml,yaml}' + + - name: Check workflows + run: go run github.com/rhysd/actionlint/cmd/actionlint@v1.7.12 diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml new file mode 100644 index 0000000..e75539b --- /dev/null +++ b/.github/workflows/quality.yml @@ -0,0 +1,123 @@ +name: Quality + +on: + merge_group: + pull_request: + branches: [main] + push: + branches: [main] + schedule: + - cron: "17 4 * * 1" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: quality-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + languages: + name: Detect languages + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + matrix: ${{ steps.detect.outputs.matrix }} + + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Build CodeQL matrix + id: detect + shell: bash + run: | + if [ -n "$(git ls-files -- '*.py')" ]; then + echo 'matrix={"include":[{"language":"actions"},{"language":"python"}]}' >> "$GITHUB_OUTPUT" + else + echo 'matrix={"include":[{"language":"actions"}]}' >> "$GITHUB_OUTPUT" + fi + + codeql: + name: CodeQL / ${{ matrix.language }} + needs: languages + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + actions: read + contents: read + security-events: write + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.languages.outputs.matrix) }} + + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Initialize CodeQL + uses: github/codeql-action/init@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4.37.3 + with: + languages: ${{ matrix.language }} + build-mode: none + queries: security-and-quality + + - name: Analyze + uses: github/codeql-action/analyze@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4.37.3 + with: + category: /language:${{ matrix.language }} + + dependency-review: + name: Dependency review + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Review dependency changes + uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5.0.0 + with: + fail-on-severity: moderate + + osv: + name: OSV scan + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + permissions: + actions: read + contents: read + security-events: write + uses: google/osv-scanner-action/.github/workflows/osv-scanner-reusable.yml@9a498708959aeaef5ef730655706c5a1df1edbc2 # v2.3.8 + + python-audit: + name: Python audit + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Install uv + if: hashFiles('dev/gui.py') != '' + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + version: "0.11.32" + python-version: "3.11" + + - name: Audit dependencies + if: hashFiles('dev/gui.py') != '' + run: uv audit --script dev/gui.py From 205c0152bc31e9ecc864490383c8addd647b59f0 Mon Sep 17 00:00:00 2001 From: Keys <70819367+keys-i@users.noreply.github.com> Date: Wed, 18 Dec 2024 18:07:00 +0530 Subject: [PATCH 2/6] update: check Python tools across platforms Run shared Python validation consistently on Linux, macOS, and Windows. --- .github/workflows/checks.yml | 58 +++++++++++++++++++++++++++-------- .github/workflows/quality.yml | 52 ++++++++++++++++++++++++++----- 2 files changed, 90 insertions(+), 20 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 52e9144..399334e 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -3,9 +3,9 @@ name: Checks on: merge_group: pull_request: - branches: [main] + branches: [main, gui] push: - branches: [main] + branches: [main, actions, gui, py] workflow_dispatch: permissions: @@ -81,8 +81,40 @@ jobs: shell: pwsh run: ./sys/windows/clean.ps1 -SelfTest + python-style: + name: Python / style + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + version: "0.11.32" + python-version: "3.11" + + - name: Find and check Python scripts + shell: bash + run: | + mapfile -t python_files < <( + git ls-files -- 'bin/_common.py' + git grep -Il '^#!/usr/bin/env -S uv run --script' -- '*.py' 'bin/*' || true + ) + if (( ${#python_files[@]} == 0 )); then + echo "No Python scripts found" + exit 0 + fi + printf 'Checking %s\n' "${python_files[@]}" + uvx ruff@0.16.0 format --check --target-version=py311 "${python_files[@]}" + uvx ruff@0.16.0 check --output-format=github --target-version=py311 "${python_files[@]}" + python: - name: Python / ${{ matrix.os }} + name: Python / runtime / ${{ matrix.os }} runs-on: ${{ matrix.os }} timeout-minutes: 10 strategy: @@ -97,24 +129,26 @@ jobs: persist-credentials: false - name: Install uv - if: hashFiles('dev/gui.py') != '' uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 with: version: "0.11.32" python-version: "3.11" - - name: Check formatting - if: hashFiles('dev/gui.py') != '' - run: uvx ruff@0.16.0 format --check --target-version=py311 dev/gui.py - - - name: Lint - if: hashFiles('dev/gui.py') != '' - run: uvx ruff@0.16.0 check --output-format=github --target-version=py311 dev/gui.py - - name: Test dashboard if: hashFiles('dev/gui.py') != '' run: uv run dev/gui.py --self-test + - name: Test command tools + shell: bash + run: | + found=0 + for script in bin/agent bin/cleaner bin/disk bin/hardware bin/slurm; do + [[ -f "$script" ]] || continue + found=1 + uv run --script "$script" --self-test + done + (( found )) || echo "No Python command tools found" + metadata: name: Repository files runs-on: ubuntu-latest diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index e75539b..3fb3fd3 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -3,9 +3,9 @@ name: Quality on: merge_group: pull_request: - branches: [main] + branches: [main, gui] push: - branches: [main] + branches: [main, actions, gui, py] schedule: - cron: "17 4 * * 1" workflow_dispatch: @@ -35,7 +35,7 @@ jobs: id: detect shell: bash run: | - if [ -n "$(git ls-files -- '*.py')" ]; then + if git grep -Il '^#!/usr/bin/env -S uv run --script' -- '*.py' 'bin/*' >/dev/null 2>&1; then echo 'matrix={"include":[{"language":"actions"},{"language":"python"}]}' >> "$GITHUB_OUTPUT" else echo 'matrix={"include":[{"language":"actions"}]}' >> "$GITHUB_OUTPUT" @@ -101,7 +101,7 @@ jobs: uses: google/osv-scanner-action/.github/workflows/osv-scanner-reusable.yml@9a498708959aeaef5ef730655706c5a1df1edbc2 # v2.3.8 python-audit: - name: Python audit + name: Python / dependencies runs-on: ubuntu-latest timeout-minutes: 10 @@ -112,12 +112,48 @@ jobs: persist-credentials: false - name: Install uv - if: hashFiles('dev/gui.py') != '' uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 with: version: "0.11.32" python-version: "3.11" - - name: Audit dependencies - if: hashFiles('dev/gui.py') != '' - run: uv audit --script dev/gui.py + - name: Audit script dependencies + shell: bash + run: | + found=0 + for script in dev/gui.py bin/agent bin/cleaner bin/disk bin/hardware bin/slurm; do + [[ -f "$script" ]] || continue + found=1 + uv audit --script "$script" + done + (( found )) || echo "No dependency-bearing Python scripts found" + + python-security: + name: Python / security lint + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + version: "0.11.32" + python-version: "3.11" + + - name: Scan Python scripts + shell: bash + run: | + mapfile -t python_files < <( + git ls-files -- 'bin/_common.py' + git grep -Il '^#!/usr/bin/env -S uv run --script' -- '*.py' 'bin/*' || true + ) + if (( ${#python_files[@]} == 0 )); then + echo "No Python scripts found" + exit 0 + fi + uvx bandit@1.9.4 --quiet --severity-level medium "${python_files[@]}" From d6be01483d4e9781be1d0cc73e10bd524f9c1f16 Mon Sep 17 00:00:00 2001 From: Keys <70819367+keys-i@users.noreply.github.com> Date: Sun, 9 Feb 2025 11:35:00 +0530 Subject: [PATCH 3/6] fix: skip empty uv caches Avoid restoring dependency caches when a Python lockfile is absent. --- .github/workflows/checks.yml | 2 ++ .github/workflows/quality.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 399334e..a60f7b4 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -97,6 +97,7 @@ jobs: with: version: "0.11.32" python-version: "3.11" + enable-cache: false - name: Find and check Python scripts shell: bash @@ -133,6 +134,7 @@ jobs: with: version: "0.11.32" python-version: "3.11" + enable-cache: false - name: Test dashboard if: hashFiles('dev/gui.py') != '' diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 3fb3fd3..26ebca7 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -116,6 +116,7 @@ jobs: with: version: "0.11.32" python-version: "3.11" + enable-cache: false - name: Audit script dependencies shell: bash @@ -144,6 +145,7 @@ jobs: with: version: "0.11.32" python-version: "3.11" + enable-cache: false - name: Scan Python scripts shell: bash From 96708ec6668760f8250ff6bdd90c2b0f26719799 Mon Sep 17 00:00:00 2001 From: Keys <70819367+keys-i@users.noreply.github.com> Date: Sun, 7 Sep 2025 16:46:00 +0530 Subject: [PATCH 4/6] add: automate Action dependency updates Open grouped weekly GitHub Actions updates without automatic merging. --- .github/dependabot.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..b8cfc26 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,20 @@ +version: 2 + +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + day: saturday + time: "17:30" + timezone: Asia/Kolkata + open-pull-requests-limit: 5 + groups: + actions-minor-patch: + patterns: + - "*" + update-types: + - minor + - patch + commit-message: + prefix: update From 1a35af63b2d457739444e4227168b4bd9173356a Mon Sep 17 00:00:00 2001 From: Keys <70819367+keys-i@users.noreply.github.com> Date: Fri, 17 Apr 2026 19:04:00 +0530 Subject: [PATCH 5/6] fix: discover Python tools in workflows Find executable tools from the repository instead of maintaining a fixed list. --- .github/workflows/checks.yml | 10 +++++----- .github/workflows/quality.yml | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index a60f7b4..1d03e7e 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -143,13 +143,13 @@ jobs: - name: Test command tools shell: bash run: | - found=0 - for script in bin/agent bin/cleaner bin/disk bin/hardware bin/slurm; do - [[ -f "$script" ]] || continue - found=1 + mapfile -t scripts < <( + git grep -Il '^#!/usr/bin/env -S uv run --script' -- 'bin/*' || true + ) + for script in "${scripts[@]}"; do uv run --script "$script" --self-test done - (( found )) || echo "No Python command tools found" + (( ${#scripts[@]} )) || echo "No Python command tools found" metadata: name: Repository files diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 26ebca7..6f57b23 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -121,13 +121,13 @@ jobs: - name: Audit script dependencies shell: bash run: | - found=0 - for script in dev/gui.py bin/agent bin/cleaner bin/disk bin/hardware bin/slurm; do - [[ -f "$script" ]] || continue - found=1 + mapfile -t scripts < <( + git grep -Il '^#!/usr/bin/env -S uv run --script' -- '*.py' 'bin/*' || true + ) + for script in "${scripts[@]}"; do uv audit --script "$script" done - (( found )) || echo "No dependency-bearing Python scripts found" + (( ${#scripts[@]} )) || echo "No dependency-bearing Python scripts found" python-security: name: Python / security lint From 61c5a8ba974cff78a918e4c388823077262d7974 Mon Sep 17 00:00:00 2001 From: Keys <70819367+keys-i@users.noreply.github.com> Date: Sat, 25 Jul 2026 17:33:00 +0530 Subject: [PATCH 6/6] fix: support macOS Bash in runtime checks Keep runtime validation compatible with the Bash version shipped by macOS. --- .github/workflows/checks.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 1d03e7e..d0e1e5d 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -143,13 +143,15 @@ jobs: - name: Test command tools shell: bash run: | - mapfile -t scripts < <( + found=0 + while IFS= read -r script; do + [[ -n "$script" ]] || continue + found=1 + uv run --script "$script" --self-test + done < <( git grep -Il '^#!/usr/bin/env -S uv run --script' -- 'bin/*' || true ) - for script in "${scripts[@]}"; do - uv run --script "$script" --self-test - done - (( ${#scripts[@]} )) || echo "No Python command tools found" + (( found )) || echo "No Python command tools found" metadata: name: Repository files