diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 2a9b7e44..3f7853c8 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -7,6 +7,11 @@ updates: - directory: / + groups: + upload-download-artifact: + patterns: + - actions/download-artifact + - actions/upload-artifact ignore: # Managed by cisagov/skeleton-generic - dependency-name: actions/cache @@ -36,6 +41,16 @@ updates: interval: weekly - directory: / + ignore: + # Managed by cisagov/skeleton-python-library + - dependency-name: build + - dependency-name: coverage + - dependency-name: coveralls + - dependency-name: pre-commit + - dependency-name: pytest-cov + - dependency-name: pytest + - dependency-name: setuptools + - dependency-name: twine package-ecosystem: pip schedule: interval: weekly diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9edabeac..3402297e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -5,6 +5,8 @@ on: # yamllint disable-line rule:truthy merge_group: types: - checks_requested + # We use the default activity types for the pull_request event as specified here: + # https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows#pull_request pull_request: push: repository_dispatch: @@ -23,7 +25,7 @@ env: PIP_CACHE_DIR: ~/.cache/pip PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit RUN_TMATE: ${{ secrets.RUN_TMATE }} - TERRAFORM_DOCS_REPO_BRANCH_NAME: improvement/support_atx_closed_markdown_headers + TERRAFORM_DOCS_REPO_BRANCH_NAME: cisagov TERRAFORM_DOCS_REPO_DEPTH: 1 TERRAFORM_DOCS_REPO_URL: https://github.com/mcdonnnj/terraform-docs.git @@ -120,22 +122,21 @@ jobs: name: Lookup Go cache directory run: | echo "dir=$(go env GOCACHE)" >> $GITHUB_OUTPUT - - uses: actions/cache@v4 + - uses: actions/cache@v5 env: - BASE_CACHE_KEY: ${{ github.job }}-${{ runner.os }}-\ - py${{ steps.setup-python.outputs.python-version }}-\ - go${{ steps.setup-go.outputs.go-version }}-\ - packer${{ steps.setup-env.outputs.packer-version }}-\ - tf${{ steps.setup-env.outputs.terraform-version }}- + BASE_CACHE_KEY: >- + ${{ github.job }}-${{ runner.os + }}-py${{ steps.setup-python.outputs.python-version + }}-go${{ steps.setup-go.outputs.go-version + }}-packer${{ steps.setup-env.outputs.packer-version + }}-tf${{ steps.setup-env.outputs.terraform-version }}- with: - # We do not use '**/setup.py' in the cache key so only the 'setup.py' - # file in the root of the repository is used. This is in case a Python - # package were to have a 'setup.py' as part of its internal codebase. - key: ${{ env.BASE_CACHE_KEY }}\ - ${{ hashFiles('**/requirements-test.txt') }}-\ - ${{ hashFiles('**/requirements.txt') }}-\ - ${{ hashFiles('**/.pre-commit-config.yaml') }}-\ - ${{ hashFiles('setup.py') }} + key: >- + ${{ env.BASE_CACHE_KEY }}${{ + hashFiles('**/requirements-test.txt')}}-${{ + hashFiles('**/requirements.txt') }}-${{ + hashFiles('**/.pre-commit-config.yaml') }}-${{ + hashFiles('pyproject.toml') }} # Note that the .terraform directory IS NOT included in the # cache because if we were caching, then we would need to use # the `-upgrade=true` option. This option blindly pulls down the @@ -151,12 +152,12 @@ jobs: - uses: hashicorp/setup-packer@v3 with: version: ${{ steps.setup-env.outputs.packer-version }} - - uses: hashicorp/setup-terraform@v3 + - uses: hashicorp/setup-terraform@v4 with: terraform_version: ${{ steps.setup-env.outputs.terraform-version }} - name: Install go-critic env: - PACKAGE_URL: github.com/go-critic/go-critic/cmd/gocritic + PACKAGE_URL: github.com/go-critic/go-critic/cmd/go-critic PACKAGE_VERSION: ${{ steps.setup-env.outputs.go-critic-version }} run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - name: Install goimports @@ -175,10 +176,13 @@ jobs: PACKAGE_VERSION: ${{ steps.setup-env.outputs.staticcheck-version }} run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} # TODO: https://github.com/cisagov/skeleton-generic/issues/165 - # We are temporarily using @mcdonnnj's forked branch of terraform-docs - # until his PR: https://github.com/terraform-docs/terraform-docs/pull/745 - # is approved. This temporary fix will allow for ATX header support when - # terraform-docs is run during linting. + # We are temporarily using a branch of @mcdonnnj's fork of terraform-docs that + # groups changes from his PRs until they are approved and merged: + # https://github.com/terraform-docs/terraform-docs/pull/745 + # https://github.com/terraform-docs/terraform-docs/pull/901 + # This temporary fix will allow for ATX header support when terraform-docs is run + # during linting and output delimiter rows with cell spacing that passes + # Markdownlint's MD060/table-column-style rule. - name: Clone ATX headers branch from terraform-docs fork run: | git clone \ @@ -193,7 +197,7 @@ jobs: -o $(go env GOPATH)/bin/terraform-docs - name: Install dependencies run: | - python -m pip install --upgrade pip setuptools wheel + python -m pip install --upgrade pip setuptools pip install --upgrade --requirement requirements-test.txt - name: Set up pre-commit hook environments run: pre-commit install-hooks @@ -202,7 +206,7 @@ jobs: - name: Setup tmate debug session uses: mxschmitt/action-tmate@v3 if: env.RUN_TMATE - test: + test-source: name: test source - py${{ matrix.python-version }} - ${{ matrix.platform }} needs: - diagnostics @@ -267,20 +271,19 @@ jobs: uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - - uses: actions/cache@v4 + - uses: actions/cache@v5 env: - BASE_CACHE_KEY: ${{ github.job }}-\ - ${{ runner.os }}-${{ runner.arch }}-\ - py${{ steps.setup-python.outputs.python-version }}- + BASE_CACHE_KEY: >- + ${{ github.job }}-${{ + runner.os }}-${{ runner.arch }}-py${{ + steps.setup-python.outputs.python-version }}- with: path: ${{ env.PIP_CACHE_DIR }} - # We do not use '**/setup.py' in the cache key so only the 'setup.py' - # file in the root of the repository is used. This is in case a Python - # package were to have a 'setup.py' as part of its internal codebase. - key: ${{ env.BASE_CACHE_KEY }}\ - ${{ hashFiles('**/requirements-test.txt') }}-\ - ${{ hashFiles('**/requirements.txt') }}-\ - ${{ hashFiles('setup.py') }} + key: >- + ${{ env.BASE_CACHE_KEY }}${{ + hashFiles('**/requirements-test.txt') }}-${{ + hashFiles('**/requirements.txt') }}${{ + hashFiles('pyproject.toml') }} restore-keys: | ${{ env.BASE_CACHE_KEY }} - name: Install dependencies @@ -307,7 +310,7 @@ jobs: runs-on: ubuntu-latest needs: - diagnostics - - test + - test-source steps: - name: Apply standard cisagov job preamble uses: cisagov/action-job-preamble@v1 @@ -343,12 +346,10 @@ jobs: - name: Setup tmate debug session uses: mxschmitt/action-tmate@v3 if: env.RUN_TMATE - build: + build-wheel: name: build wheel - py${{ matrix.python-version }} needs: - diagnostics - - lint - - test permissions: # actions/checkout needs this to fetch code contents: read @@ -401,39 +402,39 @@ jobs: uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - - uses: actions/cache@v4 + - uses: actions/cache@v5 env: - BASE_CACHE_KEY: ${{ github.job }}-${{ runner.os }}-\ - py${{ steps.setup-python.outputs.python-version }}- + BASE_CACHE_KEY: >- + ${{ github.job }}-${{ runner.os }}-py${{ + steps.setup-python.outputs.python-version }}- with: path: ${{ env.PIP_CACHE_DIR }} - # We do not use '**/setup.py' in the cache key so only the 'setup.py' - # file in the root of the repository is used. This is in case a Python - # package were to have a 'setup.py' as part of its internal codebase. - key: ${{ env.BASE_CACHE_KEY }}\ - ${{ hashFiles('**/requirements.txt') }}-\ - ${{ hashFiles('setup.py') }} + key: >- + ${{ env.BASE_CACHE_KEY }}${{ + hashFiles('**/requirements.txt') }}-${{ + hashFiles('pyproject.toml') }} restore-keys: | ${{ env.BASE_CACHE_KEY }} - name: Install build dependencies run: | - python -m pip install --upgrade pip setuptools wheel + python -m pip install --upgrade pip setuptools python -m pip install --upgrade build - name: Build artifacts run: python -m build - name: Upload artifacts - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 with: name: dist-${{ matrix.python-version }} path: dist - name: Setup tmate debug session uses: mxschmitt/action-tmate@v3 if: env.RUN_TMATE - test-build: + test-wheel: name: test built wheel - py${{ matrix.python-version }} - ${{ matrix.platform }} needs: - diagnostics - - build + - build-wheel + - test-source permissions: # actions/checkout needs this to fetch code contents: read @@ -495,23 +496,22 @@ jobs: uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - - uses: actions/cache@v4 + - uses: actions/cache@v5 env: - BASE_CACHE_KEY: ${{ github.job }}-\ - ${{ runner.os }}-${{ runner.arch }}-\ - py${{ steps.setup-python.outputs.python-version }}- + BASE_CACHE_KEY: >- + ${{ github.job }}-${{ + runner.os }}-${{ runner.arch }}-py${{ + steps.setup-python.outputs.python-version }}- with: path: ${{ env.PIP_CACHE_DIR }} - # We do not use '**/setup.py' in the cache key so only the 'setup.py' - # file in the root of the repository is used. This is in case a Python - # package were to have a 'setup.py' as part of its internal codebase. - key: ${{ env.BASE_CACHE_KEY }}\ - ${{ hashFiles('**/requirements.txt') }}-\ - ${{ hashFiles('setup.py') }} + key: >- + ${{ env.BASE_CACHE_KEY }}${{ + hashFiles('**/requirements.txt') }}-${{ + hashFiles('pyproject.toml') }} restore-keys: | ${{ env.BASE_CACHE_KEY }} - name: Retrieve the built wheel - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v8 with: name: dist-${{ matrix.python-version }} path: dist @@ -519,7 +519,7 @@ jobs: name: Get the name of the retrieved wheel (there should only be one) run: echo "wheel=$(ls dist/*whl)" >> $GITHUB_OUTPUT - name: Update core Python packages - run: python -m pip install --upgrade pip setuptools wheel + run: python -m pip install --upgrade pip setuptools - name: Install the built wheel (along with testing dependencies) run: python -m pip install ${{ steps.find-wheel.outputs.wheel }}[test] - name: Run tests diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 4e868961..cd672f6b 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -12,6 +12,8 @@ on: merge_group: types: - checks_requested + # We use the default activity types for the pull_request event as specified here: + # https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows#pull_request pull_request: # The branches here must be a subset of the ones in the push key branches: diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 580fa9c0..2b71638d 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -5,6 +5,8 @@ on: # yamllint disable-line rule:truthy merge_group: types: - checks_requested + # We use the default activity types for the pull_request event as specified here: + # https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows#pull_request pull_request: # Set a default shell for any run steps. The `-Eueo pipefail` sets errtrace, diff --git a/.github/workflows/label-prs.yml b/.github/workflows/label-prs.yml index 412cc4aa..45d317b5 100644 --- a/.github/workflows/label-prs.yml +++ b/.github/workflows/label-prs.yml @@ -2,11 +2,9 @@ name: Label pull requests on: # yamllint disable-line rule:truthy + # We use the default activity types for the pull_request event as specified here: + # https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows#pull_request pull_request: - types: - - edited - - opened - - synchronize # Set a default shell for any run steps. The `-Eueo pipefail` sets errtrace, # nounset, errexit, and pipefail. The `-x` will print all commands as they are diff --git a/.github/workflows/sync-labels.yml b/.github/workflows/sync-labels.yml index f60bc846..a8d01be1 100644 --- a/.github/workflows/sync-labels.yml +++ b/.github/workflows/sync-labels.yml @@ -87,7 +87,7 @@ jobs: - uses: actions/checkout@v6 - name: Sync repository labels if: success() - uses: crazy-max/ghaction-github-labeler@v5 + uses: crazy-max/ghaction-github-labeler@v6 with: # This is a hideous ternary equivalent so we only do a dry run unless # this workflow is triggered by the develop branch. diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 01ffcb1d..4f226e60 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -45,17 +45,17 @@ repos: # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.45.0 + rev: v0.48.0 hooks: - id: markdownlint args: - --config=.mdl_config.yaml - repo: https://github.com/rbubley/mirrors-prettier - rev: v3.6.2 + rev: v3.8.1 hooks: - id: prettier - repo: https://github.com/adrienverge/yamllint - rev: v1.37.1 + rev: v1.38.0 hooks: - id: yamllint args: @@ -63,17 +63,14 @@ repos: # GitHub Actions hooks - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.35.0 + rev: 0.37.0 hooks: - id: check-github-actions - id: check-github-workflows # pre-commit hooks - repo: https://github.com/pre-commit/pre-commit - # pre-commit v3+ dropped support for Python <3.8. Until this project and - # the build.yml workflow can migrate to Python 3.8 or newer we must - # continue to use an older version. - rev: v2.21.0 + rev: v4.5.1 hooks: - id: validate_manifest @@ -108,7 +105,7 @@ repos: # Shell script hooks - repo: https://github.com/scop/pre-commit-shfmt - rev: v3.12.0-2 + rev: v3.13.0-1 hooks: - id: shfmt args: @@ -131,12 +128,9 @@ repos: - id: shellcheck # Python hooks - # Run bandit on the "tests" tree with a configuration + # Run bandit on the "tests" tree - repo: https://github.com/PyCQA/bandit - # bandit 1.7.6 dropped support for Python <3.8. Until this project - # and the build.yml workflow can migrate to Python 3.8 or newer we - # must continue to use an older version. - rev: 1.7.5 + rev: 1.9.4 hooks: - id: bandit name: bandit (tests tree) @@ -150,10 +144,7 @@ repos: - pbr # Run bandit on everything except the "tests" tree - repo: https://github.com/PyCQA/bandit - # bandit 1.7.6 dropped support for Python <3.8. Until this project - # and the build.yml workflow can migrate to Python 3.8 or newer we - # must continue to use an older version. - rev: 1.7.5 + rev: 1.9.4 hooks: - id: bandit name: bandit (everything else) @@ -162,33 +153,29 @@ repos: - importlib-metadata<5 - pbr - repo: https://github.com/psf/black-pre-commit-mirror - rev: 25.11.0 + rev: 26.3.1 hooks: - id: black - repo: https://github.com/PyCQA/flake8 - # flake8 v6+ dropped support for Python <3.8. Until this project and - # the build.yml workflow can migrate to Python 3.8 or newer we must - # continue to use an older version. - rev: 5.0.4 + rev: 7.3.0 hooks: - id: flake8 additional_dependencies: + - dlint==0.16.0 + - flake8-bugbear==25.11.29 + - flake8-comprehensions==3.17.0 - flake8-docstrings==1.7.0 + - flake8-noqa==1.5.0 # This is necessary to read the flake8 configuration from # the pyproject.toml file. - flake8-pyproject==1.2.3 + - pep8-naming==0.15.1 - repo: https://github.com/PyCQA/isort - # isort 5.12.0 dropped support for Python <3.8. Until this project and - # the build.yml workflow can migrate to Python 3.8 or newer we must - # continue to use an older version. - rev: 5.11.5 + rev: 8.0.1 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - # mypy 1.5.0 dropped support for Python <3.8. Until this project - # and the build.yml workflow can migrate to Python 3.8 or newer we - # must continue to use an older version. - rev: v1.4.1 + rev: v1.19.1 hooks: - id: mypy # IMPORTANT: Keep type hinting-related dependencies of the @@ -222,10 +209,7 @@ repos: # - --requirement # - requirements.txt - repo: https://github.com/asottile/pyupgrade - # pyupgrade no longer supports Python 3.7 as of version 3.4.0, so - # we cannot upgrade past the 3.3.2 release: - # https://github.com/asottile/pyupgrade/blob/v3.4.0/setup.cfg#L23 - rev: v3.3.2 + rev: v3.21.2 hooks: - id: pyupgrade args: @@ -236,15 +220,10 @@ repos: # Ansible hooks # - repo: https://github.com/ansible/ansible-lint - # # ansible-lint no longer supports Python 3.7 as of version 6.0, so - # # we cannot upgrade past the 5.4.0 release: - # # https://github.com/ansible/ansible-lint/releases/tag/v6.0.0 - # # - # # But the 5.4.0 release causes a different failure because the - # # version of ansible isn't correctly pinned. The best way forward - # # is to simply comment out this pre-commit hook until we can move - # # to Python >3.7. - # rev: v25.11.1 + # # We need to stay on this version because we are still using Python 3.13 in + # # our GitHub Actions configuration. Later versions require Python 3.14 for + # # the hook to run. + # rev: v26.1.1 # hooks: # - id: ansible-lint # additional_dependencies: @@ -266,14 +245,23 @@ repos: # # Note that any changes made to this dependency must also be # # made in requirements.txt in cisagov/skeleton-packer and # # requirements-test.txt in cisagov/skeleton-ansible-role. - # - ansible-core>=2.17 + # - ansible-core>=2.17.7 # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.103.0 + rev: v1.105.0 hooks: - id: terraform_fmt - id: terraform_validate + # This needs to run after the terraform_validate hook so that any Terraform + # configurations are initialized. + - id: terraform_providers_lock + args: + - --args=-platform=darwin_amd64 + - --args=-platform=darwin_arm64 + - --args=-platform=linux_amd64 + - --args=-platform=linux_arm64 + - --hook-config=--mode=always-regenerate-lockfile # Docker hooks - repo: https://github.com/IamTheFij/docker-pre-commit diff --git a/pyproject.toml b/pyproject.toml index bca0cb8f..73893a83 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -110,30 +110,40 @@ source = "https://github.com/cisagov/pshtt" [tool.flake8] max-line-length = 80 # Select (turn on) -# * Complexity violations reported by mccabe (C) - -# http://flake8.pycqa.org/en/latest/user/error-codes.html#error-violation-codes -# * Documentation conventions compliance reported by pydocstyle (D) - -# http://www.pydocstyle.org/en/stable/error_codes.html -# * Default errors and warnings reported by pycodestyle (E and W) - +# * C: Complexity violations reported by mccabe - +# https://flake8.pycqa.org/en/latest/user/error-codes.html#error-violation-codes +# * C4: Default errors and warnings reported by flake8-comprehensions - +# https://github.com/adamchainz/flake8-comprehensions#rules +# * D: Documentation conventions compliance reported by pydocstyle - +# https://github.com/PyCQA/pydocstyle/blob/master/docs/error_codes.rst +# * DUO: Default errors and warnings reported by dlint - +# https://github.com/dlint-py/dlint/tree/master/docs +# * E: Default errors reported by pycodestyle - # https://pycodestyle.readthedocs.io/en/latest/intro.html#error-codes -# * Default errors reported by pyflakes (F) - -# http://flake8.pycqa.org/en/latest/glossary.html#term-pyflakes -# * Default warnings reported by flake8-bugbear (B) - +# * F: Default errors reported by pyflakes - +# https://flake8.pycqa.org/en/latest/glossary.html#term-pyflakes +# * N: Default errors and warnings reported by pep8-naming - +# https://github.com/PyCQA/pep8-naming#error-codes +# * NQA: Default errors and warnings reported by flake8-noqa - +# https://github.com/plinss/flake8-noqa#error-codes +# * W: Default warnings reported by pycodestyle - +# https://pycodestyle.readthedocs.io/en/latest/intro.html#error-codes +# * B: Default warnings reported by flake8-bugbear - # https://github.com/PyCQA/flake8-bugbear#list-of-warnings -# * The B950 flake8-bugbear opinionated warning - +# * B950: Bugbear opinionated warning for line too long - # https://github.com/PyCQA/flake8-bugbear#opinionated-warnings -select = ["C", "D", "E", "F", "W", "B", "B950"] -# Ignore flake8's default warning about "whitespace before ':'" as it -# is not PEP 8 compliant and conflicts with black's styling. -# -# Ignore flake8's default warning about maximum line length, which has -# a hard stop at the configured value. Instead we use -# flake8-bugbear's B950, which allows up to 10% overage. -# -# Also ignore flake8's warning about line breaks before binary -# operators. It no longer agrees with PEP8. See, for example, here: -# https://github.com/ambv/black/issues/21. Guido agrees here: -# https://github.com/python/peps/commit/c59c4376ad233a62ca4b3a6060c81368bd21e85b. +select = ["C", "C4", "D", "DUO", "E", "F", "N", "NQA", "W", "B", "B950"] +# Ignore +# * E203: pycodestyle's default warning about whitespace before ':' because Black enforces +# an equal amount of whitespace around slice operators (':'). +# * E501: pycodestyle's default warning about maximum line length, which has a hard stop +# at the configured value. Instead we use flake8-bugbear's B950, which +# allows up to 10% overage. +# * W503: pycodestyle's warning about line breaks before binary operators. It no longer +# agrees with PEP8. See, for example, here: +# https://github.com/ambv/black/issues/21 +# Guido agrees here: +# https://github.com/python/peps/commit/c59c4376ad233a62ca4b3a6060c81368bd21e85b extend-ignore = ["E203", "E501", "W503"] [tool.isort] diff --git a/requirements.txt b/requirements.txt index 8b75fe95..75ae4870 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -# Note: Add any additional requirements to setup.py's install_requires field +# Note: Add any additional requirements to pyproject.toml's +# dependencies entry --editable . -wheel diff --git a/setup-env b/setup-env index cf9d787d..bd8d5e23 100755 --- a/setup-env +++ b/setup-env @@ -271,7 +271,7 @@ fi pyenv local "${env_name}" # Upgrade pip and friends -python3 -m pip install --upgrade pip setuptools wheel +python3 -m pip install --upgrade pip setuptools # Find a requirements file (if possible) and install for req_file in "requirements-dev.txt" "requirements-test.txt" "requirements.txt"; do diff --git a/src/pshtt/cli.py b/src/pshtt/cli.py index 9ee8e195..aaffe710 100644 --- a/src/pshtt/cli.py +++ b/src/pshtt/cli.py @@ -1,7 +1,9 @@ """pshtt ("pushed") is a tool to test domains for HTTPS best practices. Usage: - pshtt (INPUT ...) [--output OUTFILE] [--sorted] [--json] [--markdown] [--debug] [--timeout TIMEOUT] [--user-agent AGENT] [--cache-third-parties DIR] [--ca-file PATH] [--pt-int-ca-file PATH] + pshtt (INPUT ...) [--output OUTFILE] [--sorted] [--json] [--markdown] [--debug] + [--timeout TIMEOUT] [--user-agent AGENT] [--cache-third-parties DIR] + [--ca-file PATH] [--pt-int-ca-file PATH] pshtt (-h | --help) Options: @@ -13,9 +15,11 @@ -d --debug Print debug output. -u --user-agent=AGENT Override user agent. -t --timeout=TIMEOUT Override timeout (in seconds). - -c --cache-third-parties=DIR Cache third party data, and what directory to cache it in. + -c --cache-third-parties=DIR Cache third party data, and what + directory to cache it in. -f --ca-file=PATH Specify custom CA bundle (PEM format) - -p --pt-int-ca-file=PATH Specify public trust CA bundle with intermediates (PEM format) + -p --pt-int-ca-file=PATH Specify public trust CA bundle with + intermediates (PEM format) Notes: If the first INPUT ends with .csv, domains will be read from CSV. diff --git a/src/pshtt/pshtt.py b/src/pshtt/pshtt.py index aa56a79d..07bfeced 100644 --- a/src/pshtt/pshtt.py +++ b/src/pshtt/pshtt.py @@ -9,7 +9,6 @@ import os from pathlib import Path # Python3 import re -import sys from urllib import parse as urlparse # Third-Party Libraries @@ -37,7 +36,11 @@ # disabled. Commented next line due to pylint warning that urllib3 is # not in requests.packages # requests.packages.urllib3.disable_warnings() -urllib3.disable_warnings() +# +# Without the noqa comment flake8 generates a DUO131 error because +# disabling this warning allows for the possibility of insecure +# connections. +urllib3.disable_warnings() # noqa: DUO131 # Default, overrideable via --user-agent USER_AGENT = "pshtt, https scanning" @@ -316,7 +319,8 @@ def basic_check(endpoint): "sslv3 alert handshake failure" in str(err) or "Unexpected EOF" in str(err) ): logging.exception( - "%s: Error completing TLS handshake usually due to required client authentication.", + "%s: Error completing TLS handshake usually due to required " + "client authentication.", endpoint.url, ) utils.debug("%s: %s", endpoint.url, err) @@ -415,7 +419,8 @@ def basic_check(endpoint): # Run SSLyze to see if there are any errors if endpoint.protocol == "https": https_check(endpoint) - # Double-check in case sslyze failed the first time, but the regular conneciton succeeded + # Double-check in case sslyze failed the first time, but the + # regular connection succeeded if endpoint.live is False and req is not None: logging.warning( "%s: Trying sslyze again since it connected once already.", endpoint.url @@ -424,7 +429,8 @@ def basic_check(endpoint): endpoint.https_valid = True https_check(endpoint) if endpoint.live is False: - # sslyze failed so back everything out and don't continue analyzing the existing response + # sslyze failed so back everything out and don't + # continue analyzing the existing response req = None endpoint.https_valid = False endpoint.https_full_connection = False @@ -450,7 +456,8 @@ def basic_check(endpoint): ip, ) except Exception: - # if the socket has already closed, it will throw an exception, but this is just best effort, so ignore it + # if the socket has already closed, it will throw an exception, + # but this is just best effort, so ignore it logging.exception("Error closing socket") # Endpoint is live, analyze the response. @@ -558,12 +565,14 @@ def basic_check(endpoint): base_original == base_eventual ) and (subdomain_original != subdomain_eventual) - # If we were able to make the first redirect, but not the ultimate redirect, - # and if the immediate redirect is external, then it's accurate enough to - # say that the eventual redirect is the immediate redirect, since you're capturing - # the domain it's going to. - # This also avoids "punishing" the domain for configuration issues of the site - # it redirects to. + # If we were able to make the first redirect, but not the + # ultimate redirect, and if the immediate redirect is + # external, then it's accurate enough to say that the + # eventual redirect is the immediate redirect, since you're + # capturing the domain it's going to. + # + # This also avoids "punishing" the domain for configuration + # issues of the site it redirects to. elif endpoint.redirect_immediately_to_external: endpoint.redirect_eventually_to = endpoint.redirect_immediately_to endpoint.redirect_eventually_to_https = ( @@ -707,7 +716,7 @@ def https_check(endpoint): ) scanner.queue_scan(scan_request) # Retrieve results from generator object - scan_result = [x for x in scanner.get_results()][0] + scan_result = list(scanner.get_results())[0] cert_plugin_result = scan_result.scan_commands_results[ ScanCommand.CERTIFICATE_INFO ] @@ -719,7 +728,7 @@ def https_check(endpoint): ) scanner.queue_scan(scan_request) # Consume the generator object and retrieve the first result - scan_result = [x for x in scanner.get_results()][0] + scan_result = list(scanner.get_results())[0] cert_plugin_result = scan_result.scan_commands_results[ ScanCommand.CERTIFICATE_INFO ] @@ -805,7 +814,9 @@ def https_check(endpoint): # SAN(s) are checked as part of _certificate_matches_hostname which # called as part of leaf_certificate_subject_matches_hostname if ( - not certificate_deployment.leaf_certificate_subject_matches_hostname + # flake8 complains that this line is too long, but I + # don't see a way to split it up. + not certificate_deployment.leaf_certificate_subject_matches_hostname # noqa: B950 ): endpoint.https_bad_hostname = True @@ -848,7 +859,9 @@ def https_check(endpoint): if endpoint.https_self_signed_cert is False and ( endpoint.https_cert_chain_len < 2 ): - # *** TODO check that it is not a bad hostname and that the root cert is trusted before suggesting that it is an intermediate cert issue. + # *** TODO - check that it is not a bad hostname and that + # the root cert is trusted before suggesting that it is an + # intermediate cert issue. endpoint.https_missing_intermediate_cert = True has_verified_cert_chain = True for certificate_deployment in cert_plugin_result.certificate_deployments: @@ -856,7 +869,8 @@ def https_check(endpoint): has_verified_cert_chain = False if not has_verified_cert_chain: logging.warning( - "%s: Untrusted certificate chain, probably due to missing intermediate certificate.", + "%s: Untrusted certificate chain, probably due to " + "missing intermediate certificate.", endpoint.url, ) utils.debug( @@ -865,7 +879,8 @@ def https_check(endpoint): endpoint.https_cert_chain_len, ) elif custom_trust is True and public_trust is False: - # recheck public trust using custom public trust store with manually added intermediate certificates + # recheck public trust using custom public trust store + # with manually added intermediate certificates if PT_INT_CA_FILE is not None: try: cert_plugin_result = None @@ -883,7 +898,7 @@ def https_check(endpoint): ) scanner.queue_scan(scan_request) # Consume the generator object and retrieve the first result - scan_result = [x for x in scanner.get_results()][0] + scan_result = list(scanner.get_results())[0] cert_plugin_result = scan_result.scan_commands_results[ ScanCommand.CERTIFICATE_INFO ] @@ -900,7 +915,8 @@ def https_check(endpoint): public_trust = True endpoint.https_public_trusted = public_trust logging.warning( - "%s: Trusted by special public trust store with intermediate certificates.", + "%s: Trusted by special public trust store " + "with intermediate certificates.", endpoint.url, ) except Exception: @@ -1580,10 +1596,7 @@ def load_preload_pending(): return [] # TODO: abstract Py 2/3 check out to utils - if sys.version_info[0] < 3: - raw = request.content - else: - raw = str(request.content, "utf-8") + raw = str(request.content, "utf-8") pending_json = json.loads(raw) @@ -1602,7 +1615,10 @@ def load_preload_list(): utils.debug("Fetching Chrome preload list from source...", divider=True) # Downloads the chromium preloaded domain list and sets it to a global set - file_url = "https://chromium.googlesource.com/chromium/src/+/main/net/http/transport_security_state_static.json?format=TEXT" + file_url = ( + "https://chromium.googlesource.com/chromium/src/+/main/net/http/" + "transport_security_state_static.json?format=TEXT" + ) try: request = requests.get(file_url, timeout=TIMEOUT) diff --git a/tests/test_utils.py b/tests/test_utils.py index 5cffa3de..f6397821 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -23,7 +23,7 @@ def test_with_empty_filename(self): Should raise a `FileNotFoundError` """ - with self.assertRaises(FileNotFoundError): # noqa + with self.assertRaises(FileNotFoundError): with smart_open(""): pass