diff --git a/.github/workflows/agentic-pr-review.yml b/.github/workflows/agentic-pr-review.yml new file mode 100644 index 0000000..3926092 --- /dev/null +++ b/.github/workflows/agentic-pr-review.yml @@ -0,0 +1,546 @@ +name: Reusable Push + Agentic PR Review + +on: + pull_request: + types: + - opened + - reopened + - synchronize + - ready_for_review + +permissions: + contents: read + +concurrency: + group: reusable-push-agentic-pr-review-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + pr-labeling: + runs-on: ubuntu-latest + if: github.event.action == 'opened' + permissions: + issues: write + pull-requests: write + contents: read + steps: + - name: Assign PR Labels + uses: splunk-soar-connectors/.github/.github/actions/pr-labeling@main + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + jira-user: ${{ secrets.JIRA_USER }} + jira-api-key: ${{ secrets.JIRA_API_KEY }} + repo-name: ${{ github.repository }} + pr-number: ${{ github.event.pull_request.number }} + + detect-app-type: + runs-on: ubuntu-latest + outputs: + is_sdkfied: ${{ steps.detect.outputs.is_sdkfied }} + uv_lock_directory: ${{ steps.detect.outputs.uv_lock_directory }} + steps: + - name: Check out app repo + uses: actions/checkout@v4 + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} + + - name: Detect App Type + id: detect + run: | + echo "Detecting app type..." + + # Find uv.lock file and get its parent directory + UV_LOCK_PATH=$(find . -name "uv.lock" -type f | head -1) + + if [ -n "$UV_LOCK_PATH" ]; then + UV_LOCK_DIR=$(dirname "$UV_LOCK_PATH") + echo "Found uv.lock in directory: $UV_LOCK_DIR" + echo "This is an SDKfied app" + echo "is_sdkfied=true" >> $GITHUB_OUTPUT + echo "uv_lock_directory=$UV_LOCK_DIR" >> $GITHUB_OUTPUT + else + echo "No uv.lock found" + echo "This is a traditional app" + echo "is_sdkfied=false" >> $GITHUB_OUTPUT + echo "uv_lock_directory=" >> $GITHUB_OUTPUT + fi + + echo "App detection completed" + shell: bash + + pytest: + runs-on: ubuntu-latest + needs: detect-app-type + if: needs.detect-app-type.outputs.is_sdkfied == 'true' + permissions: + contents: read + env: + UV_LOCK_DIRECTORY: ${{ needs.detect-app-type.outputs.uv_lock_directory }} + steps: + - name: Check out app repo + uses: actions/checkout@v4 + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + + - name: Set up SDKfied app environment + uses: splunk-soar-connectors/.github/.github/actions/sdkfied-app-setup@main + with: + uv_lock_directory: ${{ env.UV_LOCK_DIRECTORY }} + + - name: Run pytest + run: uv run --project "$UV_LOCK_DIRECTORY" python -m pytest + + pre-commit: + runs-on: ubuntu-latest + steps: + - name: Setup Environment + uses: splunk-soar-connectors/.github/.github/actions/env-setup@main + + - name: Pre-commit + uses: splunk-soar-connectors/.github/.github/actions/pre-commit@main + + semantic-release-preview: + runs-on: ubuntu-latest + permissions: + contents: write + issues: write + pull-requests: write + steps: + - name: Check out app repo + uses: actions/checkout@v4 + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Check out shared workflow files + uses: actions/checkout@v4 + with: + repository: splunk-soar-connectors/.github + ref: main + path: dotgithub + + - name: Set up NodeJS + uses: actions/setup-node@v4 + with: + node-version: "24" + + - name: Install semantic-release dependencies + run: npm install --global semantic-release @semantic-release/exec @semantic-release/git conventional-changelog-conventionalcommits + + - name: Prepare release configuration + run: | + cp dotgithub/.releaserc.json .releaserc.json + + - name: Run Semantic Release as a dry run + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_HEAD_REF: ${{ github.event.pull_request.head.ref }} + run: | + unset GITHUB_ACTIONS GITHUB_REF GITHUB_EVENT_NAME + semantic-release --dry-run --no-ci --branches "$PR_HEAD_REF" --tag-format '${version}' + + - name: Comment release notes on the PR + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + breadcrumb='' + message='Merging this PR will not result in a release.' + + if [[ -s release_version.txt ]]; then + version_number=$(")) | .id' \ + | head -1) + + if [[ -n "$comment_id" ]]; then + gh api --method PATCH "repos/$GH_REPO/issues/comments/$comment_id" -f body="$comment_body" + else + gh api --method POST "repos/$GH_REPO/issues/$PR_NUMBER/comments" -f body="$comment_body" + fi + + compile: + runs-on: + - codebuild-integration-tests-${{ github.run_id }}-${{ github.run_attempt }} + - image:custom-linux-875003031410.dkr.ecr.us-west-2.amazonaws.com/soar-connectors/pytest:f7150dbb7f347d35f8f4bb285d36985ecd4cf231 + needs: detect-app-type + env: + IS_SDKFIED: ${{ needs.detect-app-type.outputs.is_sdkfied }} + UV_LOCK_DIRECTORY: ${{ needs.detect-app-type.outputs.uv_lock_directory }} + steps: + - name: Check out app repo + uses: actions/checkout@v4 + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} + + - name: Setup SDKfied App Environment + if: env.IS_SDKFIED == 'true' + uses: splunk-soar-connectors/.github/.github/actions/sdkfied-app-setup@main + with: + uv_lock_directory: ${{ env.UV_LOCK_DIRECTORY }} + + - name: Compile Application + uses: splunk-soar-connectors/.github/.github/actions/compile-app@main + with: + current_phantom_ip: ${{ vars.PHANTOM_INSTANCE_CURRENT_VERSION_IP }} + next_phantom_ip: ${{ vars.PHANTOM_INSTANCE_NEXT_OL8_VERSION_IP }} + previous_phantom_ip: ${{ vars.PHANTOM_INSTANCE_PREVIOUS_VERSION_IP }} + phantom_username: ${{ vars.PHANTOM_USERNAME }} + phantom_password: password + is_sdkfied: ${{ needs.detect-app-type.outputs.is_sdkfied }} + uv_lock_directory: ${{ needs.detect-app-type.outputs.uv_lock_directory }} + + build: + # connector-template intentionally uses a sentinel app ID and is not a publishable app. + if: github.event.repository.name != 'connector-template' + runs-on: + - codebuild-integration-tests-${{ github.run_id }}-${{ github.run_attempt }} + - image:custom-linux-875003031410.dkr.ecr.us-west-2.amazonaws.com/soar-connectors/pytest:f7150dbb7f347d35f8f4bb285d36985ecd4cf231 + needs: [compile, detect-app-type] + env: + IS_SDKFIED: ${{ needs.detect-app-type.outputs.is_sdkfied }} + UV_LOCK_DIRECTORY: ${{ needs.detect-app-type.outputs.uv_lock_directory }} + steps: + - uses: actions/checkout@v4 + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} + + - name: Build Application + if: env.IS_SDKFIED == 'false' + uses: splunk-soar-connectors/.github/.github/actions/build-app@main + + - name: Setup SDKfied App Environment + if: env.IS_SDKFIED == 'true' + uses: splunk-soar-connectors/.github/.github/actions/sdkfied-app-setup@main + with: + uv_lock_directory: ${{ env.UV_LOCK_DIRECTORY }} + + - name: Build SDK app + if: env.IS_SDKFIED == 'true' + run: | + cd "${{ env.UV_LOCK_DIRECTORY }}" + echo "Building SDKfied app using soarapps CLI" + uv run soarapps package build . --output-file /tmp/${{ github.event.repository.name }}.tgz + echo "SDKfied app build completed successfully" + + - name: Upload app tar file as artifact + uses: actions/upload-artifact@v4 + with: + name: app-tar + path: /tmp/${{ github.event.repository.name }}.tgz + + test-setup: + runs-on: ubuntu-latest + needs: [build, detect-app-type] + env: + IS_SDKFIED: ${{ needs.detect-app-type.outputs.is_sdkfied }} + UV_LOCK_DIRECTORY: ${{ needs.detect-app-type.outputs.uv_lock_directory }} + outputs: + fips_compliant: ${{ steps.set-outputs.outputs.fips_compliant }} + publisher: ${{ steps.set-outputs.outputs.publisher }} + steps: + - name: Check out actions repo + uses: actions/checkout@v4 + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} + + - name: Setup Python 3.13 for SDKfied apps + if: env.IS_SDKFIED == 'true' + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Setup SDKfied App Environment + if: env.IS_SDKFIED == 'true' + uses: splunk-soar-connectors/.github/.github/actions/sdkfied-app-setup@main + with: + uv_lock_directory: ${{ env.UV_LOCK_DIRECTORY }} + + - name: Create temporary manifest for SDKfied app + if: env.IS_SDKFIED == 'true' + run: | + uv run --project "${{ env.UV_LOCK_DIRECTORY }}" soarapps manifests create temp_app.json "${{ env.UV_LOCK_DIRECTORY }}" + echo "Created temporary manifest: temp_app.json" + shell: bash + + - name: Set sanity tests environment variables + uses: splunk-soar-connectors/.github/.github/actions/test-setup@main + + - name: Clean up temporary manifest for SDKfied app + if: env.IS_SDKFIED == 'true' + run: | + rm -f temp_app.json + echo "Deleted temporary manifest: temp_app.json" + shell: bash + + - name: Set outputs + id: set-outputs + run: | + echo "${{ env.publisher }}" + echo publisher="${{ env.publisher }}" >> $GITHUB_OUTPUT + echo "fips_compliant=${{ env.fips_compliant }}" >> $GITHUB_OUTPUT + + test-coverage: + runs-on: + - codebuild-integration-tests-${{ github.run_id }}-${{ github.run_attempt }} + - image:custom-linux-875003031410.dkr.ecr.us-west-2.amazonaws.com/soar-connectors/pytest:f7150dbb7f347d35f8f4bb285d36985ecd4cf231 + needs: [test-setup, detect-app-type] + env: + IS_SDKFIED: ${{ needs.detect-app-type.outputs.is_sdkfied }} + UV_LOCK_DIRECTORY: ${{ needs.detect-app-type.outputs.uv_lock_directory }} + if: ${{ needs.test-setup.outputs.publisher == 'Splunk' }} + continue-on-error: true + steps: + - name: Check out actions repo + uses: actions/checkout@v4 + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} + + - name: Setup SDKfied App Environment + if: env.IS_SDKFIED == 'true' + uses: splunk-soar-connectors/.github/.github/actions/sdkfied-app-setup@main + with: + uv_lock_directory: ${{ env.UV_LOCK_DIRECTORY }} + + - name: Create temporary manifest for SDKfied app + if: env.IS_SDKFIED == 'true' + run: | + uv run --project "${{ env.UV_LOCK_DIRECTORY }}" soarapps manifests create temp_app.json "${{ env.UV_LOCK_DIRECTORY }}" + echo "Created temporary manifest: temp_app.json" + shell: bash + + - name: Run Test Coverage + uses: splunk-soar-connectors/.github/.github/actions/test-coverage@main + with: + app_repo: ${{ github.event.repository.name }} + + - name: Clean up temporary manifest for SDKfied app + if: env.IS_SDKFIED == 'true' + run: | + rm -f temp_app.json + echo "Deleted temporary manifest: temp_app.json" + shell: bash + + sanity-test: + runs-on: + - codebuild-integration-tests-${{ github.run_id }}-${{ github.run_attempt }} + - image:custom-linux-875003031410.dkr.ecr.us-west-2.amazonaws.com/soar-connectors/pytest:f7150dbb7f347d35f8f4bb285d36985ecd4cf231 + needs: test-setup + if: ${{ needs.test-setup.outputs.publisher == 'Splunk' }} + strategy: + fail-fast: false + matrix: + include: + - version: "next_ol8" + ip: ${{ vars.PHANTOM_INSTANCE_NEXT_OL8_VERSION_IP }} + - version: "next_ol9" + ip: ${{ vars.PHANTOM_INSTANCE_NEXT_OL9_VERSION_IP }} + - version: "next_amzn2023" + ip: ${{ vars.PHANTOM_INSTANCE_NEXT_AMZN2023_VERSION_IP }} + - version: "previous" + ip: ${{ vars.PHANTOM_INSTANCE_PREVIOUS_VERSION_IP }} + - version: "cloud" + ip: ${{ vars.PHANTOM_INSTANCE_CLOUD_HOST }} + - version: "cloud_next" + ip: ${{ vars.PHANTOM_INSTANCE_CLOUD_NEXT_HOST }} + env: + NUM_TEST_RETRIES: ${{ vars.NUM_TEST_RETRIES }} + VAULT_ROLE: ${{ vars.VAULT_ROLE }} + PHANTOM_INSTANCE_CURRENT_FIPS_VERSION_IP: ${{ vars.PHANTOM_INSTANCE_CURRENT_FIPS_VERSION_IP }} + PLAYBOOK_EXCLUDE_REGEX: ${{ vars.PLAYBOOK_EXCLUDE_REGEX }} + + steps: + - uses: actions/checkout@v4 + + - name: Download app tar file + uses: actions/download-artifact@v4 + with: + name: app-tar + + - name: Check version compatibility + id: version-check + uses: splunk-soar-connectors/.github/.github/actions/version-check@main + with: + tarball_name: ${{ github.event.repository.name }}.tgz + phantom_ip: ${{ matrix.ip }} + + - name: Run Sanity Tests + if: steps.version-check.outputs.compatible == 'true' + uses: splunk-soar-connectors/.github/.github/actions/sanity-tests@main + with: + phantom_ip: ${{ matrix.ip }} + version: ${{ matrix.version }} + app_repo: ${{ github.event.repository.name }} + automation_broker: ${{ vars.AUTOMATION_BROKER_NAME }} + automation_broker_next: ${{ vars.AUTOMATION_BROKER_NEXT_NAME }} + + - name: Upload Sanity Test Results + uses: actions/upload-artifact@v4 + if: always() && steps.version-check.outputs.compatible == 'true' + with: + name: sanity-test-results-${{ matrix.version }} + path: ${{ github.workspace }}/test-results/ + retention-days: 1 + + aws-sanity-test: + runs-on: + - codebuild-integration-tests-${{ github.run_id }}-${{ github.run_attempt }} + - image:custom-linux-875003031410.dkr.ecr.us-west-2.amazonaws.com/soar-connectors/pytest:f7150dbb7f347d35f8f4bb285d36985ecd4cf231 + needs: test-setup + if: ${{ needs.test-setup.outputs.publisher == 'Splunk' && contains(github.event.repository.name, 'aws') }} + strategy: + fail-fast: false + matrix: + include: + - version: "aws_current" + - version: "aws_previous" + - version: "aws_next" + env: + AWS_PHANTOM_INSTANCE_CURRENT_VERSION_IP: ${{ secrets.AWS_PHANTOM_INSTANCE_CURRENT_VERSION_IP }} + AWS_PHANTOM_INSTANCE_PREVIOUS_VERSION_IP: ${{ secrets.AWS_PHANTOM_INSTANCE_PREVIOUS_VERSION_IP }} + AWS_PHANTOM_INSTANCE_NEXT_VERSION_IP: ${{ secrets.AWS_PHANTOM_INSTANCE_NEXT_VERSION_IP }} + AWS_PHANTOM_USERNAME: ${{ vars.AWS_PHANTOM_USERNAME }} + AWS_PHANTOM_PASSWORD: ${{ secrets.AWS_PHANTOM_PASSWORD }} + NUM_TEST_RETRIES: ${{ vars.NUM_TEST_RETRIES }} + VAULT_ROLE: ${{ vars.VAULT_ROLE }} + + steps: + - name: Resolve AWS Phantom IP + id: resolve-ip + shell: bash + run: | + case "${{ matrix.version }}" in + aws_current) ip="$AWS_PHANTOM_INSTANCE_CURRENT_VERSION_IP" ;; + aws_previous) ip="$AWS_PHANTOM_INSTANCE_PREVIOUS_VERSION_IP" ;; + aws_next) ip="$AWS_PHANTOM_INSTANCE_NEXT_VERSION_IP" ;; + esac + if [[ -z "$ip" ]]; then + echo "::error::AWS Phantom instance IP is not set for version '${{ matrix.version }}'. Set the corresponding repository secret." + exit 1 + fi + echo "ip=$ip" >> "$GITHUB_OUTPUT" + echo "AWS_PHANTOM_URL=https://$ip" >> "$GITHUB_ENV" + - uses: actions/checkout@v4 + + - name: Download app tar file + uses: actions/download-artifact@v4 + with: + name: app-tar + + - name: Run Sanity Tests + uses: splunk-soar-connectors/.github/.github/actions/sanity-tests@main + with: + phantom_ip: ${{ steps.resolve-ip.outputs.ip }} + version: ${{ matrix.version }} + app_repo: ${{ github.event.repository.name }} + + - name: Upload Sanity Test Results + uses: actions/upload-artifact@v4 + if: always() + with: + name: sanity-test-results-${{ matrix.version }} + path: ${{ github.workspace }}/test-results/ + retention-days: 1 + + aggregate-sanity-results: + runs-on: ubuntu-latest + needs: [sanity-test, aws-sanity-test, test-setup] + if: always() && needs.test-setup.outputs.publisher == 'Splunk' + steps: + - name: Download All Sanity Test Artifacts + uses: actions/download-artifact@v4 + with: + pattern: sanity-test-results-* + path: downloaded-artifacts/ + merge-multiple: false + + - name: Aggregate Test Results + uses: splunk-soar-connectors/.github/.github/actions/aggregate-sanity-results@main + with: + artifacts_path: downloaded-artifacts + + integration-test: + runs-on: + - codebuild-integration-tests-${{ github.run_id }}-${{ github.run_attempt }} + - image:custom-linux-875003031410.dkr.ecr.us-west-2.amazonaws.com/soar-connectors/pytest:f7150dbb7f347d35f8f4bb285d36985ecd4cf231 + needs: test-setup + if: ${{ needs.test-setup.outputs.publisher == 'Splunk' }} + env: + PHANTOM_INSTANCE_IP: ${{ vars.PHANTOM_INSTANCE_CURRENT_VERSION_IP }} + PHANTOM_INSTANCE_IP_FIPS: ${{ vars.PHANTOM_INSTANCE_CURRENT_FIPS_VERSION_IP }} + NUM_TEST_RETRIES: ${{ vars.NUM_TEST_RETRIES }} + VAULT_ROLE: ${{ vars.VAULT_ROLE }} + steps: + - uses: actions/checkout@v4 + + - name: Download app tar file + uses: actions/download-artifact@v4 + with: + name: app-tar + + - name: Run Integration Tests + uses: splunk-soar-connectors/.github/.github/actions/integration-tests@main + with: + fips_compliant: ${{ needs.test-setup.outputs.fips_compliant }} + app_repo: ${{ github.event.repository.name }} + + agentic-pr-review: + name: Agentic PR review + needs: + - pr-labeling + - detect-app-type + - pytest + - pre-commit + - semantic-release-preview + - compile + - build + - test-setup + - test-coverage + - sanity-test + - aws-sanity-test + - aggregate-sanity-results + - integration-test + if: always() && github.event.pull_request.draft == false + permissions: + actions: read + checks: read + contents: read + pull-requests: read + issues: read + uses: splunk-soar-connectors/AgenticPRReview/.github/workflows/pr-review.yml@sanchaga/agenticprreviewbot + with: + repo: ${{ github.repository }} + pr_number: ${{ github.event.pull_request.number }} + circuit_base_url: ${{ vars.CIRCUIT_BASE_URL }} + circuit_model: ${{ vars.CIRCUIT_MODEL }} + circuit_token_url: ${{ vars.CIRCUIT_TOKEN_URL }} + previous_pipeline_results_json: >- + {"jobs":{"pre-commit":{"result":"${{ needs['pre-commit'].result }}","url":"${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"},"compile":{"result":"${{ needs.compile.result }}","url":"${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"},"build":{"result":"${{ needs.build.result }}","url":"${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"},"semantic-release-preview":{"result":"${{ needs['semantic-release-preview'].result }}","url":"${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"}}} + publish_comments: true + secrets: + AI_REVIEW_APP_ID: ${{ secrets.AI_REVIEW_APP_ID }} + AI_REVIEW_INSTALLATION_ID: ${{ secrets.AI_REVIEW_INSTALLATION_ID }} + AI_REVIEW_PRIVATE_KEY: ${{ secrets.AI_REVIEW_PRIVATE_KEY }} + CIRCUIT_CLIENT_ID: ${{ secrets.CIRCUIT_CLIENT_ID }} + CIRCUIT_CLIENT_SECRET: ${{ secrets.CIRCUIT_CLIENT_SECRET }} + CIRCUIT_APP_KEY: ${{ secrets.CIRCUIT_APP_KEY }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a448a96..1385ad7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,13 +27,13 @@ repos: - id: check-json - id: check-yaml - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.15.22 + rev: v0.16.1 hooks: - id: ruff args: [ "--fix", "--unsafe-fixes"] # Allow unsafe fixes (ruff pretty strict about what it can fix) - id: ruff-format - repo: https://github.com/djlint/djLint - rev: v1.42.1 + rev: v1.43.2 hooks: - id: djlint-reformat-django - id: djlint-django @@ -61,7 +61,7 @@ repos: exclude: "README.md" # Central hooks - repo: https://github.com/phantomcyber/dev-cicd-tools - rev: v2.2.8 + rev: v2.2.9 hooks: - id: build-docs language: python diff --git a/release_notes/unreleased.md b/release_notes/unreleased.md index fbcb2fd..824fc3d 100644 --- a/release_notes/unreleased.md +++ b/release_notes/unreleased.md @@ -1 +1,11 @@ **Unreleased** +* Block and lookup actions now account for ZIA allowlist entries that override the advanced-threat denylist. +* Group membership updates now fetch the user immediately before writing and compare existing memberships by group ID. +* URL list actions now convert internationalized hostnames to IDNA2008 A-labels before sending them to ZIA. +* URL category actions now use incremental list updates that preserve concurrent entries and existing keyword configuration. +* The get departments action now encodes its search and pagination query parameters through the HTTP client. +* Authentication now returns a controlled error when the ZIA response omits its required session cookie. +* Unblock and unallow actions now remove only shared-list entries recorded as added by the current connector asset. +* Unexpected scalar JSON and malformed rate-limit response bodies now return controlled action errors. +* Unparseable Retry-After values now preserve the original rate-limit failure instead of being treated as success. +* URL category updates and user listing now inherit the connector's finite request timeout. diff --git a/requirements.txt b/requirements.txt index e69de29..3e62c4d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -0,0 +1 @@ +idna>=3.7,<4 diff --git a/zscaler_connector.py b/zscaler_connector.py index 2a2ea4f..91df143 100644 --- a/zscaler_connector.py +++ b/zscaler_connector.py @@ -23,6 +23,7 @@ import time from urllib.parse import quote +import idna import phantom.app as phantom import phantom.rules as phantom_rules import requests @@ -190,6 +191,14 @@ def _process_json_response(self, r, action_result): # Please specify the status codes here if 200 <= r.status_code < 399: + if not isinstance(resp_json, (dict, list)): + return RetVal( + action_result.set_status( + phantom.APP_ERROR, + f"Unexpected JSON response type from server: {type(resp_json).__name__}. Status Code: {r.status_code}", + ), + None, + ) return RetVal(phantom.APP_SUCCESS, resp_json) # You should process the error returned in the json @@ -316,13 +325,14 @@ def _make_rest_call_helper(self, *args, **kwargs): if self._response.status_code == 429 and self._retry_rest_call != 0: # Rate limit exceeded try: retry_time = self._response.json()["Retry-After"] - except KeyError: - self.debug_print("KeyError") + except (KeyError, TypeError, ValueError): + self.debug_print("Missing or malformed Retry-After in rate-limit response") return ret_val, response self.debug_print(f"Retry Time: {retry_time}") seconds_to_wait = self._parse_retry_time(retry_time) if seconds_to_wait is None or seconds_to_wait < 0: - return retry_time, response + self.debug_print(f"Unparseable Retry-After value: {retry_time}") + return ret_val, response self.send_progress(f"Exceeded rate limit: Retrying after {retry_time}") time.sleep(seconds_to_wait) self._retry_rest_call -= 1 # reduce the number of retries @@ -359,7 +369,13 @@ def _init_session(self): return self.set_status(phantom.APP_ERROR, f"Error starting Zscaler session: {action_result.get_message()}") else: self.save_progress("Successfully started Zscaler session") - self._headers = {"cookie": self._response.headers["Set-Cookie"].split(";")[0].strip()} + set_cookie = self._response.headers.get("Set-Cookie") + if not set_cookie: + return self.set_status( + phantom.APP_ERROR, + "Error starting Zscaler session: authentication response did not include a Set-Cookie header", + ) + self._headers = {"cookie": set_cookie.split(";")[0].strip()} return phantom.APP_SUCCESS def _deinit_session(self): @@ -404,6 +420,49 @@ def _filter_endpoints(self, action_result, to_add, existing, action, name): return RetVal(action_result.set_status(phantom.APP_SUCCESS, msg), None) return RetVal(phantom.APP_SUCCESS, endpoints) + def _ownership_ledger(self): + ledger = self._state.get("installation_added_entries") + if not isinstance(ledger, dict): + ledger = {} + self._state["installation_added_entries"] = ledger + return ledger + + def _owned_entries(self, key): + entries = self._ownership_ledger().get(key) + return entries if isinstance(entries, list) else [] + + def _filter_owned_removals(self, action_result, ledger_key, endpoints, action): + if action != "REMOVE_FROM_LIST": + return endpoints, [] + + owned_entries = set(self._owned_entries(ledger_key)) + owned = [endpoint for endpoint in endpoints if endpoint in owned_entries] + foreign = [endpoint for endpoint in endpoints if endpoint not in owned_entries] + if not owned: + summary = action_result.set_summary({}) + summary["updated"] = [] + summary["ignored"] = endpoints + summary["skipped_foreign"] = foreign + action_result.set_status( + phantom.APP_ERROR, + "None of these endpoints were added by this connector asset; refusing to remove shared entries created elsewhere", + ) + return None, foreign + return owned, foreign + + def _update_ownership_ledger(self, key, endpoints, action): + entries = set(self._owned_entries(key)) + if action == "ADD_TO_LIST": + entries.update(endpoints) + else: + entries.difference_update(endpoints) + + ledger = self._ownership_ledger() + if entries: + ledger[key] = sorted(entries) + else: + ledger.pop(key, None) + def _get_blocklist(self, action_result): return self._make_rest_call_helper("/api/v1/security/advanced", action_result) @@ -421,6 +480,21 @@ def _amend_blocklist(self, action_result, endpoints, action): if phantom.is_fail(ret_val) or filtered_endpoints is None: return ret_val + if action == "ADD_TO_LIST": + ret_val, allowlist_response = self._get_allowlist(action_result) + if phantom.is_fail(ret_val): + return ret_val + allowlisted = sorted(set(filtered_endpoints) & set(allowlist_response.get("whitelistUrls", []))) + if allowlisted: + return action_result.set_status( + phantom.APP_ERROR, + f"Endpoint(s) {allowlisted} are on the ZIA allowlist, which overrides the denylist. Remove them from the allowlist first.", + ) + + filtered_endpoints, skipped_foreign = self._filter_owned_removals(action_result, "blocklist", filtered_endpoints, action) + if filtered_endpoints is None: + return action_result.get_status() + params = {"action": action} data = {"blacklistUrls": filtered_endpoints} ret_val, _response = self._make_rest_call_helper( @@ -428,12 +502,14 @@ def _amend_blocklist(self, action_result, endpoints, action): ) if phantom.is_fail(ret_val) and self._response.status_code != 204: return ret_val + self._update_ownership_ledger("blocklist", filtered_endpoints, action) ret_val = self._activate_config(action_result) if phantom.is_fail(ret_val): return ret_val summary = action_result.set_summary({}) summary["updated"] = filtered_endpoints summary["ignored"] = list(set(endpoints) - set(filtered_endpoints)) + summary["skipped_foreign"] = skipped_foreign # Encode the unicode IP or URL strings summary["updated"] = [element for element in summary["updated"]] summary["ignored"] = [element for element in summary["ignored"]] @@ -457,6 +533,10 @@ def _amend_allowlist(self, action_result, endpoints, action): if phantom.is_fail(ret_val) or filtered_endpoints is None: return ret_val + filtered_endpoints, skipped_foreign = self._filter_owned_removals(action_result, "allowlist", filtered_endpoints, action) + if filtered_endpoints is None: + return action_result.get_status() + if action == "ADD_TO_LIST": to_add_endpoints = list(set(self._allowlist + filtered_endpoints)) else: @@ -467,6 +547,7 @@ def _amend_allowlist(self, action_result, endpoints, action): if phantom.is_fail(ret_val): return ret_val + self._update_ownership_ledger("allowlist", filtered_endpoints, action) ret_val = self._activate_config(action_result) if phantom.is_fail(ret_val): return ret_val @@ -475,6 +556,7 @@ def _amend_allowlist(self, action_result, endpoints, action): summary = action_result.set_summary({}) summary["updated"] = filtered_endpoints summary["ignored"] = list(set(endpoints) - set(filtered_endpoints)) + summary["skipped_foreign"] = skipped_foreign # Encode the unicode IP or URL strings summary["updated"] = [element for element in summary["updated"]] summary["ignored"] = [element for element in summary["ignored"]] @@ -505,25 +587,33 @@ def _check_category(self, action_result, endpoints, category, action): return self._filter_endpoints(action_result, endpoints, urls, action, "Category") + def _category_delta_payload(self, category, urls, db_categorized_urls): + payload = {"urls": urls, "dbCategorizedUrls": db_categorized_urls} + for field in ("configuredName", "superCategory", "keywords", "keywordsRetainingParentCategory"): + if field in category: + payload[field] = category[field] + return payload + def _amend_category(self, action_result, endpoints, category, action): ret_val, filtered_endpoints = self._check_category(action_result, endpoints, category, action) if phantom.is_fail(ret_val) or filtered_endpoints is None: return ret_val + ledger_key = f"category:{self._category['id']}" + filtered_endpoints, skipped_foreign = self._filter_owned_removals(action_result, ledger_key, filtered_endpoints, action) + if filtered_endpoints is None: + return action_result.get_status() + params = {"action": action} - data = { - "configuredName": self._category.get("configuredName"), - "keywordsRetainingParentCategory": self._category.get("keywordsRetainingParentCategory", []), - "urls": [], - "dbCategorizedUrls": filtered_endpoints, - } + data = self._category_delta_payload(self._category, [], filtered_endpoints) ret_val, response = self._make_rest_call_helper( - "/api/v1/urlCategories/{}".format(self._category["id"]), action_result, data=data, method="put", params=params, timeout=None + "/api/v1/urlCategories/{}".format(self._category["id"]), action_result, data=data, method="put", params=params ) if phantom.is_fail(ret_val): return ret_val + self._update_ownership_ledger(ledger_key, filtered_endpoints, action) ret_val = self._activate_config(action_result) if phantom.is_fail(ret_val): return ret_val @@ -531,6 +621,7 @@ def _amend_category(self, action_result, endpoints, category, action): summary = action_result.set_summary({}) summary["updated"] = filtered_endpoints summary["ignored"] = list(set(endpoints) - set(filtered_endpoints)) + summary["skipped_foreign"] = skipped_foreign # Encode the unicode IP or URL strings summary["updated"] = [element for element in summary["updated"]] summary["ignored"] = [element for element in summary["ignored"]] @@ -645,11 +736,17 @@ def _lookup_endpoint(self, action_result, endpoints): if phantom.is_fail(ret_val): return ret_val + ret_val, allowlist_response = self._get_allowlist(action_result) + if phantom.is_fail(ret_val): + return ret_val + + blocklist = set(blocklist_response.get("blacklistUrls", [])) + allowlist = set(allowlist_response.get("whitelistUrls", [])) for e in endpoints: - if e in blocklist_response.get("blacklistUrls", []): - [response[i].update({"blocklisted": True}) for i, item in enumerate(response) if item["url"] == e] - else: - [response[i].update({"blocklisted": False}) for i, item in enumerate(response) if item["url"] == e] + effectively_blocked = e in blocklist and e not in allowlist + for item in response: + if item.get("url") == e: + item["blocklisted"] = effectively_blocked action_result.update_data(response) @@ -801,9 +898,34 @@ def _truncate_protocol(self, endpoints): endpoints[i] = endpoints[i][(len("http://")) :] elif endpoints[i].lower().startswith("https://"): endpoints[i] = endpoints[i][(len("https://")) :] + endpoints[i] = self._to_ascii_hostname(endpoints[i]) return endpoints + def _to_ascii_hostname(self, endpoint): + host_and_port, separator, path = endpoint.partition("/") + if host_and_port.startswith("[") or host_and_port.count(":") > 1: + return endpoint + + host, port_separator, port = host_and_port.partition(":") + labels = [] + for label in host.split("."): + if not label or "*" in label or label.isascii(): + labels.append(label) + continue + try: + labels.append(idna.encode(label, uts46=True).decode("ascii")) + except idna.IDNAError: + labels.append(label) + + normalized_host = ".".join(labels) + normalized_endpoint = normalized_host + if port_separator: + normalized_endpoint += f"{port_separator}{port}" + if separator: + normalized_endpoint += f"{separator}{path}" + return normalized_endpoint + def _check_for_overlength(self, action_result, endpoints): """This function checks whether the length of each url is not more than 1024 @@ -877,7 +999,7 @@ def _handle_get_users(self, param): users = [] while True: params["pageSize"] = min(limit, ZSCALER_MAX_PAGESIZE) - ret_val, get_users = self._make_rest_call_helper("/api/v1/users", action_result, params=params, timeout=None) + ret_val, get_users = self._make_rest_call_helper("/api/v1/users", action_result, params=params) if phantom.is_fail(ret_val): return action_result.get_status() for user in get_users: @@ -949,14 +1071,16 @@ def _handle_add_group_user(self, param): ret_val, group_id = self._validate_integer(action_result, param["group_id"], "group_id") if phantom.is_fail(ret_val): return action_result.get_status() - ret_val, user_response = self._make_rest_call_helper(f"/api/v1/users/{user_id}", action_result) + ret_val, group_response = self._make_rest_call_helper(f"/api/v1/groups/{group_id}", action_result) if phantom.is_fail(ret_val): return action_result.get_status() - ret_val, group_response = self._make_rest_call_helper(f"/api/v1/groups/{group_id}", action_result) + # The users API only exposes a full-object PUT, so fetch the user as + # late as possible to narrow the lost-update window. + ret_val, user_response = self._make_rest_call_helper(f"/api/v1/users/{user_id}", action_result) if phantom.is_fail(ret_val): return action_result.get_status() summary = action_result.update_summary({}) - if group_response in user_response["groups"]: + if group_response.get("id") in [item.get("id") for item in user_response["groups"]]: summary["message"] = "User already in group" action_result.add_data(group_response) return action_result.set_status(phantom.APP_SUCCESS, "User already in group") @@ -1109,18 +1233,13 @@ def _get_category_details(self, id, action_result): return phantom.APP_SUCCESS, response def _add_to_category(self, data, parent_data, cat_details, category_id, action_result): - new_data = cat_details.get("urls", []) - new_data.extend(data) - if new_data: - cat_details["urls"] = new_data - - new_parent_data = cat_details.get("dbCategorizedUrls", []) - new_parent_data.extend(parent_data) - if new_parent_data: - cat_details["dbCategorizedUrls"] = new_parent_data - + payload = self._category_delta_payload(cat_details, data, parent_data) ret_val, response = self._make_rest_call_helper( - f"/api/v1/urlCategories/{quote(str(category_id), safe='')}", action_result, data=cat_details, method="put" + f"/api/v1/urlCategories/{quote(str(category_id), safe='')}", + action_result, + data=payload, + method="put", + params={"action": "ADD_TO_LIST"}, ) if phantom.is_fail(ret_val): return ret_val, response @@ -1208,22 +1327,16 @@ def _handle_remove_category_ip(self, param): def _remove_from_category(self, data, parent_data, cat_details, category_id, action_result): data_set = set(data) - new_data = [] - for point in cat_details.get("urls", []): - if point not in data_set: - new_data.append(point) - + urls_to_remove = [point for point in cat_details.get("urls", []) if point in data_set] parent_data_set = set(parent_data) - new_parent_data = [] - for point in cat_details.get("dbCategorizedUrls", []): - if point not in parent_data_set: - new_parent_data.append(point) - - cat_details["urls"] = new_data - cat_details["dbCategorizedUrls"] = new_parent_data - + parent_urls_to_remove = [point for point in cat_details.get("dbCategorizedUrls", []) if point in parent_data_set] + payload = self._category_delta_payload(cat_details, urls_to_remove, parent_urls_to_remove) ret_val, response = self._make_rest_call_helper( - f"/api/v1/urlCategories/{quote(str(category_id), safe='')}", action_result, data=cat_details, method="put" + f"/api/v1/urlCategories/{quote(str(category_id), safe='')}", + action_result, + data=payload, + method="put", + params={"action": "REMOVE_FROM_LIST"}, ) if phantom.is_fail(ret_val): return ret_val, response @@ -1499,15 +1612,12 @@ def _handle_get_departments(self, param): action_result = self.add_action_result(ActionResult(dict(param))) name = param.get("name") - page_size = param.get("pageSize") - page_num = param.get("page", 1) - - endpoint = f"/api/v1/departments?page={page_num}&pageSize={page_size}" + params = {"page": param.get("page", 1), "pageSize": param.get("pageSize", 100)} if name: - endpoint = f"/api/v1/departments?page={page_num}&pageSize={page_size}&search={name}&limitSearch=true" + params.update({"search": name, "limitSearch": True}) - ret_val, response = self._make_rest_call_helper(endpoint, action_result) + ret_val, response = self._make_rest_call_helper("/api/v1/departments", action_result, params=params) if phantom.is_fail(ret_val): return action_result.get_status()