diff --git a/.dockerignore b/.dockerignore index 786ad94..b551615 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,4 +5,4 @@ Dockerfile LICENSE README.md -setup.cfg \ No newline at end of file +tests/ \ No newline at end of file diff --git a/.github/workflows/dogfood.yml b/.github/workflows/dogfood.yml new file mode 100644 index 0000000..1bc9fa0 --- /dev/null +++ b/.github/workflows/dogfood.yml @@ -0,0 +1,52 @@ +# This workflow tests the workbench-agent by scanning its own repository. +## This workflow runs manually on whatever branch its pointed at. +name: Scan a Branch! +on: workflow_dispatch + +jobs: + fossid_scan: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + +# Zipping the code uploads a single file for Workbench to extract. + - name: ZIP up the code + run: | + zip -r $GITHUB_WORKSPACE/code.zip . -x \ + '*.tmp' '*.temp' '*.bak' \ + '*.cache' '*.db' '*.idx' \ + '*.log' '*.txt' '*.event' \ + '*.sample' '*.demo' '*.example' \ + '*.sql' '*.hprof' '*.dmp' \ + '.gitignore' '.dockerignore' \ + '.git/*' '.github/*' + +# Since we're running with Python, install the dependencies. +# This is not needed when running the container image. + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Workbench Agent Dependencies + run: pip install -e . + +# We omit License Extraction to speed up the scan process. +# Built-In Env Vars map Branches to Scans in Workbench for this repo's Project. +# Additionally, Delta Scan reduces scan time by only scanning new files. + - name: Scan the ZIP + run: | + python workbench-agent.py \ + --api_url ${{ secrets.WORKBENCH_URL }} \ + --api_user ${{ secrets.WORKBENCH_USER }} \ + --api_token ${{ secrets.WORKBENCH_TOKEN }} \ + --project_code $GITHUB_REPOSITORY \ + --scan_code $GITHUB_REPOSITORY/$GITHUB_REF_NAME \ + --path $GITHUB_WORKSPACE/code.zip \ + --reuse_identifications \ + --chunked_upload \ + --identification_reuse_type specific_project \ + --specific_code $GITHUB_REPOSITORY \ + --run_dependency_analysis \ + --delta_only \ No newline at end of file diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml index 81637db..be7046d 100644 --- a/.github/workflows/linters.yml +++ b/.github/workflows/linters.yml @@ -5,11 +5,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 - - name: Code linters + - name: Code formatting check run: | - pip install -r requirements.txt - pycodestyle workbench-agent.py - pylint --errors-only --rcfile .pylintrc workbench-agent.py + pip install -e .[dev] + black --check workbench-agent.py api/ diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..adeb9ea --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,44 @@ +# .github/workflows/tests.yml +name: Run Tests + +# Run on pushes and pull requests +on: + push: + paths-ignore: + - '*.md' + - '.gitignore' + pull_request: + paths-ignore: + - '*.md' + - '.gitignore' + +jobs: + unit_tests: + name: Unit Tests (Python ${{ matrix.python-version }}) + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.9', '3.10', '3.11', '3.12'] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e .[dev,test] + + - name: Run unit tests + run: | + pytest tests/unit/ -v --tb=short + + - name: Run tests with coverage (Python 3.11 only) + if: matrix.python-version == '3.11' + run: | + pip install coverage pytest-cov + pytest tests/unit/ --cov=api --cov-report=xml --cov-report=term-missing \ No newline at end of file diff --git a/.gitignore b/.gitignore index 53f0526..56247d1 100755 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,10 @@ Thumbs.db # Log file -log-agent.txt \ No newline at end of file +log-agent.txt + +# Sample Code +inspiration/ +test-commands.txt +original-wb-agent.py +workbench-agent-log.txt \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index e1b736e..52ca4d6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM cgr.dev/chainguard/python:latest-dev as builder WORKDIR /app -COPY requirements.txt . -RUN pip install -r requirements.txt --user +COPY pyproject.toml . +RUN pip install -e . --user FROM cgr.dev/chainguard/python:latest WORKDIR /app diff --git a/README.md b/README.md index d4305e5..77e835f 100755 --- a/README.md +++ b/README.md @@ -235,4 +235,4 @@ Linting Run pylint in order reveal possible issues: ```bash pylint workbench-agent.py -``` +``` \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..dc4fb78 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,113 @@ +[build-system] +requires = ["setuptools>=45", "wheel", "setuptools_scm[toml]>=6.2"] + +[project] +name = "workbench-agent" +version = "0.8.0" +description = "FossID Workbench Agent - Modular API client for automated scanning" +requires-python = ">=3.9" +dependencies = [ + "requests", + "python-dotenv", +] + +[project.optional-dependencies] +dev = [ + "black", +] +test = [ + "pytest>=7.0.0", + "pytest-mock>=3.6.1", + "requests>=2.25.1", +] + +[tool.black] +line-length = 100 +target-version = ['py39'] +include = '\.pyi?$' +exclude = ''' +/( + \.eggs + | \.git + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | _build + | buck-out + | build + | dist + | inspiration +)/ +''' + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.isort] +profile = "black" +line_length = 100 +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true +ensure_newline_before_comments = true +skip = ["inspiration"] + +[tool.mypy] +python_version = "3.9" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = false +disallow_incomplete_defs = false +check_untyped_defs = true +disallow_untyped_decorators = false +no_implicit_optional = true +warn_redundant_casts = true +warn_unused_ignores = true +warn_no_return = true +warn_unreachable = true +strict_equality = true +exclude = [ + "inspiration/", + "build/", + "dist/" +] + +[tool.coverage.run] +source = ["api"] +omit = [ + "*/tests/*", + "*/inspiration/*", + "*/__pycache__/*" +] + +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "def __repr__", + "if self.debug:", + "if settings.DEBUG", + "raise AssertionError", + "raise NotImplementedError", + "if 0:", + "if __name__ == .__main__.:" +] + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = ["test_*.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] +addopts = [ + "-v", + "--tb=short", + "--strict-markers" +] +markers = [ + "unit: Unit tests", + "integration: Integration tests", + "slow: Slow-running tests" +] + + \ No newline at end of file diff --git a/requirements.txt b/requirements.txt deleted file mode 100755 index 787c11a..0000000 --- a/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -requests -python-dotenv -pycodestyle -pylint -black diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index d3c0fc6..0000000 --- a/setup.cfg +++ /dev/null @@ -1,5 +0,0 @@ -[pycodestyle] -count = False -ignore = E1,E23,E722,W503 -max-line-length = 120 -statistics = True diff --git a/src/workbench_agent.egg-info/PKG-INFO b/src/workbench_agent.egg-info/PKG-INFO new file mode 100644 index 0000000..6bb7b31 --- /dev/null +++ b/src/workbench_agent.egg-info/PKG-INFO @@ -0,0 +1,15 @@ +Metadata-Version: 2.4 +Name: workbench-agent +Version: 0.7.1 +Summary: FossID Workbench Agent - Modular API client for automated scanning +Requires-Python: >=3.9 +License-File: LICENSE +Requires-Dist: requests +Requires-Dist: python-dotenv +Provides-Extra: dev +Requires-Dist: black; extra == "dev" +Provides-Extra: test +Requires-Dist: pytest>=7.0.0; extra == "test" +Requires-Dist: pytest-mock>=3.6.1; extra == "test" +Requires-Dist: requests>=2.25.1; extra == "test" +Dynamic: license-file diff --git a/src/workbench_agent.egg-info/SOURCES.txt b/src/workbench_agent.egg-info/SOURCES.txt new file mode 100644 index 0000000..d0b3b5b --- /dev/null +++ b/src/workbench_agent.egg-info/SOURCES.txt @@ -0,0 +1,50 @@ +.dockerignore +.gitignore +.pylintrc +Dockerfile +LICENSE +README.md +pyproject.toml +workbench-agent.py +.github/workflows/build-image.yml +.github/workflows/dogfood.yml +.github/workflows/linters.yml +.github/workflows/tests.yml +api/__init__.py +api/projects_api.py +api/scans_api.py +api/upload_api.py +api/workbench_api.py +api/helpers/__init__.py +api/helpers/api_base.py +api/helpers/exceptions.py +api/helpers/process_waiters.py +api/helpers/project_scan_checks.py +api/helpers/status_checkers.py +api/helpers/upload_helpers.py +src/workbench_agent/__init__.py +src/workbench_agent.egg-info/PKG-INFO +src/workbench_agent.egg-info/SOURCES.txt +src/workbench_agent.egg-info/dependency_links.txt +src/workbench_agent.egg-info/requires.txt +src/workbench_agent.egg-info/top_level.txt +src/workbench_agent/api/__init__.py +src/workbench_agent/api/projects_api.py +src/workbench_agent/api/scans_api.py +src/workbench_agent/api/upload_api.py +src/workbench_agent/api/workbench_api.py +src/workbench_agent/api/helpers/__init__.py +src/workbench_agent/api/helpers/api_base.py +src/workbench_agent/api/helpers/process_waiters.py +src/workbench_agent/api/helpers/project_scan_checks.py +src/workbench_agent/api/helpers/status_checkers.py +src/workbench_agent/api/helpers/upload_helpers.py +src/workbench_agent/utilities/__init__.py +src/workbench_agent/utilities/error_handling.py +src/workbench_agent/utilities/exceptions.py +tests/__init__.py +tests/unit/__init__.py +tests/unit/api/__init__.py +tests/unit/api/test_projects_api.py +tests/unit/api/test_scans_api.py +tests/unit/api/test_workbench_api.py \ No newline at end of file diff --git a/src/workbench_agent.egg-info/dependency_links.txt b/src/workbench_agent.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/workbench_agent.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/src/workbench_agent.egg-info/requires.txt b/src/workbench_agent.egg-info/requires.txt new file mode 100644 index 0000000..13c1a06 --- /dev/null +++ b/src/workbench_agent.egg-info/requires.txt @@ -0,0 +1,10 @@ +requests +python-dotenv + +[dev] +black + +[test] +pytest>=7.0.0 +pytest-mock>=3.6.1 +requests>=2.25.1 diff --git a/src/workbench_agent.egg-info/top_level.txt b/src/workbench_agent.egg-info/top_level.txt new file mode 100644 index 0000000..7bbfa1a --- /dev/null +++ b/src/workbench_agent.egg-info/top_level.txt @@ -0,0 +1 @@ +workbench_agent diff --git a/src/workbench_agent/__init__.py b/src/workbench_agent/__init__.py new file mode 100644 index 0000000..239b2f2 --- /dev/null +++ b/src/workbench_agent/__init__.py @@ -0,0 +1,11 @@ +""" +FossID Workbench Agent - Modular API client for automated scanning +""" + +__version__ = "0.8.0" + +# Import main API class +from .api import WorkbenchAPI + +# Keep backward compatibility by creating an alias +Workbench = WorkbenchAPI \ No newline at end of file diff --git a/src/workbench_agent/api/__init__.py b/src/workbench_agent/api/__init__.py new file mode 100644 index 0000000..347ffa5 --- /dev/null +++ b/src/workbench_agent/api/__init__.py @@ -0,0 +1,10 @@ +""" +API modules for FossID Workbench Agent +""" + +from .projects_api import ProjectsAPI +from .scans_api import ScansAPI +from .upload_api import UploadAPI +from .workbench_api import WorkbenchAPI + +__all__ = ["ProjectsAPI", "ScansAPI", "UploadAPI", "WorkbenchAPI"] \ No newline at end of file diff --git a/src/workbench_agent/api/handlers/blind_scan.py b/src/workbench_agent/api/handlers/blind_scan.py new file mode 100644 index 0000000..7fd5f12 --- /dev/null +++ b/src/workbench_agent/api/handlers/blind_scan.py @@ -0,0 +1,135 @@ +import logging +import argparse +from typing import Dict, Any + +from ..workbench_api import WorkbenchAPI +from ...utilities.scan_workflows import ( + perform_blind_scan, + upload_scan_content, + run_kb_scan, + run_dependency_analysis, + wait_for_scan_completion, + wait_for_dependency_analysis_completion, + collect_and_save_results, + cleanup_temp_file +) +from ...utilities.error_handling import handler_error_wrapper +from ...utilities.cli_wrapper import CliWrapper +from ...exceptions import ValidationError, FileSystemError, ProcessError + +logger = logging.getLogger("workbench-agent") + + +@handler_error_wrapper +def handle_blind_scan(workbench: WorkbenchAPI, params: argparse.Namespace) -> bool: + """ + Handler for the 'blind-scan' command. Uses FossID CLI to generate file hashes, + uploads the hash file, runs KB scan, optional dependency analysis, and collects results. + + Args: + workbench: The Workbench API client instance + params: Command line parameters + + Returns: + bool: True if the operation completed successfully + + Raises: + ValidationError: If required parameters are missing or invalid + FileSystemError: If specified paths don't exist + ProcessError: If CLI execution fails + """ + logger.info("--- Starting BLIND SCAN Command ---") + + # Validate scan parameters + if not params.path: + raise ValidationError("A path must be provided for the blind-scan command.") + + # Resolve project and scan (find or create) - matching inspiration pattern + if getattr(params, 'use_name_resolution', False): + # Name-based resolution + project_code = workbench.resolve_project(params.project_name, create_if_missing=True) + scan_code, scan_id = workbench.resolve_scan( + scan_name=params.scan_name, + project_name=params.project_name, + create_if_missing=True, + params=params + ) + else: + # Legacy code-based approach + scan_code = params.scan_code # For legacy, use the scan_code directly + project_code, scan_id = workbench.prepare_project_and_scan( + project_identifier=params.project_code, + scan_identifier=params.scan_code, + params=params + ) + + # Initialize CLI wrapper + cli_wrapper = CliWrapper( + cli_path=getattr(params, 'cli_path', '/usr/bin/fossid-cli'), + config_path=getattr(params, 'config_path', '/etc/fossid.conf') + ) + + # Determine if dependency analysis should be included in hash generation + include_da_in_hash = getattr(params, 'run_dependency_analysis', False) or getattr(params, 'run_only_dependency_analysis', False) + + # Perform blind scan to generate hashes + hash_file_path = perform_blind_scan( + cli_wrapper=cli_wrapper, + path=params.path, + run_dependency_analysis=include_da_in_hash + ) + + try: + # Upload the generated hash file + upload_scan_content( + workbench=workbench, + scan_code=params.scan_code, + path=hash_file_path, + chunked_upload=getattr(params, 'chunked_upload', False) + ) + + # Determine what scans to run + run_kb = not getattr(params, 'run_only_dependency_analysis', False) + run_da = getattr(params, 'run_dependency_analysis', False) or getattr(params, 'run_only_dependency_analysis', False) + + # Calculate timeout in minutes + timeout_minutes = (getattr(params, 'scan_number_of_tries', 960) * + getattr(params, 'scan_wait_time', 30)) // 60 + + # Build scan options + scan_options = { + "limit": getattr(params, 'limit', 10), + "sensitivity": getattr(params, 'sensitivity', 10), + "auto_identification_detect_declaration": getattr(params, 'auto_identification_detect_declaration', False), + "auto_identification_detect_copyright": getattr(params, 'auto_identification_detect_copyright', False), + "auto_identification_resolve_pending_ids": getattr(params, 'auto_identification_resolve_pending_ids', False), + "delta_only": getattr(params, 'delta_only', False), + "reuse_identifications": getattr(params, 'reuse_identifications', False), + "identification_reuse_type": getattr(params, 'identification_reuse_type', 'any'), + "specific_code": getattr(params, 'specific_code', None), + "advanced_match_scoring": getattr(params, 'advanced_match_scoring', True), + "match_filtering_threshold": getattr(params, 'match_filtering_threshold', -1) + } + + # Run KB scan if requested + if run_kb: + run_kb_scan(workbench, params.scan_code, scan_options) + wait_for_scan_completion(workbench, params.scan_code, timeout_minutes) + + # Run dependency analysis if requested and not already included in hash generation + if run_da and not include_da_in_hash: + run_dependency_analysis(workbench, params.scan_code) + wait_for_dependency_analysis_completion(workbench, params.scan_code, timeout_minutes) + + # Collect and save results + results = collect_and_save_results(workbench, params.scan_code, params) + + logger.info(f"Blind scan command completed successfully. Found {len(results)} license entries.") + return True + + finally: + # Always cleanup the temporary hash file + cleanup_success = cleanup_temp_file(hash_file_path) + if not cleanup_success: + logger.warning(f"Failed to cleanup temporary hash file: {hash_file_path}") + logger.warning("You may need to manually remove this file.") diff --git a/src/workbench_agent/api/handlers/scan.py b/src/workbench_agent/api/handlers/scan.py new file mode 100644 index 0000000..66514cb --- /dev/null +++ b/src/workbench_agent/api/handlers/scan.py @@ -0,0 +1,117 @@ +import logging +import argparse +from typing import Dict, Any + +from ..workbench_api import WorkbenchAPI +from ...utilities.scan_workflows import ( + upload_scan_content, + extract_archives, + run_kb_scan, + run_dependency_analysis, + wait_for_scan_completion, + wait_for_dependency_analysis_completion, + collect_and_save_results +) +from ...utilities.error_handling import handler_error_wrapper +from ...exceptions import ValidationError, FileSystemError + +logger = logging.getLogger("workbench-agent") + + +@handler_error_wrapper +def handle_scan(workbench: WorkbenchAPI, params: argparse.Namespace) -> bool: + """ + Handler for the 'scan' command. Uploads code files directly, runs KB scan, + optional dependency analysis, and collects results. + + Args: + workbench: The Workbench API client instance + params: Command line parameters + + Returns: + bool: True if the operation completed successfully + + Raises: + ValidationError: If required parameters are missing or invalid + FileSystemError: If specified paths don't exist + """ + logger.info("--- Starting SCAN Command ---") + + # Validate scan parameters + if not params.path: + raise ValidationError("A path must be provided for the scan command.") + + # Resolve project and scan (find or create) - matching inspiration pattern + if getattr(params, 'use_name_resolution', False): + # Name-based resolution + project_code = workbench.resolve_project(params.project_name, create_if_missing=True) + scan_code, scan_id = workbench.resolve_scan( + scan_name=params.scan_name, + project_name=params.project_name, + create_if_missing=True, + params=params + ) + else: + # Legacy code-based approach + scan_code = params.scan_code # For legacy, use the scan_code directly + project_code, scan_id = workbench.prepare_project_and_scan( + project_identifier=params.project_code, + scan_identifier=params.scan_code, + params=params + ) + + # Upload content if path is provided + if params.path: + upload_scan_content( + workbench=workbench, + scan_code=params.scan_code, + path=params.path, + chunked_upload=getattr(params, 'chunked_upload', False) + ) + + # Extract archives after upload + extract_archives( + workbench=workbench, + scan_code=params.scan_code, + recursive=getattr(params, 'recursively_extract_archives', False), + jar_extraction=getattr(params, 'jar_file_extraction', False) + ) + + # Determine what scans to run + run_kb = not getattr(params, 'run_only_dependency_analysis', False) + run_da = getattr(params, 'run_dependency_analysis', False) or getattr(params, 'run_only_dependency_analysis', False) + + # Calculate timeout in minutes + timeout_minutes = (getattr(params, 'scan_number_of_tries', 960) * + getattr(params, 'scan_wait_time', 30)) // 60 + + # Build scan options + scan_options = { + "limit": getattr(params, 'limit', 10), + "sensitivity": getattr(params, 'sensitivity', 10), + "auto_identification_detect_declaration": getattr(params, 'auto_identification_detect_declaration', False), + "auto_identification_detect_copyright": getattr(params, 'auto_identification_detect_copyright', False), + "auto_identification_resolve_pending_ids": getattr(params, 'auto_identification_resolve_pending_ids', False), + "delta_only": getattr(params, 'delta_only', False), + "reuse_identifications": getattr(params, 'reuse_identifications', False), + "identification_reuse_type": getattr(params, 'identification_reuse_type', 'any'), + "specific_code": getattr(params, 'specific_code', None), + "advanced_match_scoring": getattr(params, 'advanced_match_scoring', True), + "match_filtering_threshold": getattr(params, 'match_filtering_threshold', -1) + } + + # Run KB scan if requested + if run_kb: + run_kb_scan(workbench, params.scan_code, scan_options) + wait_for_scan_completion(workbench, params.scan_code, timeout_minutes) + + # Run dependency analysis if requested + if run_da: + run_dependency_analysis(workbench, params.scan_code) + wait_for_dependency_analysis_completion(workbench, params.scan_code, timeout_minutes) + + # Collect and save results + results = collect_and_save_results(workbench, params.scan_code, params) + + logger.info(f"Scan command completed successfully. Found {len(results)} license entries.") + return True diff --git a/src/workbench_agent/api/helpers/__init__.py b/src/workbench_agent/api/helpers/__init__.py new file mode 100644 index 0000000..6a54b69 --- /dev/null +++ b/src/workbench_agent/api/helpers/__init__.py @@ -0,0 +1,3 @@ +""" +API helper modules for FossID Workbench Agent +""" \ No newline at end of file diff --git a/src/workbench_agent/api/helpers/api_base.py b/src/workbench_agent/api/helpers/api_base.py new file mode 100644 index 0000000..e34efd7 --- /dev/null +++ b/src/workbench_agent/api/helpers/api_base.py @@ -0,0 +1,163 @@ +import json +import logging +import requests +from typing import Dict, Any +from ...exceptions import ( + ApiError, + NetworkError, + AuthenticationError, + ScanNotFoundError, + ProjectNotFoundError, + ScanExistsError, + ProjectExistsError, +) +from .process_waiters import ProcessWaiters +from .status_checkers import StatusCheckers + +logger = logging.getLogger("workbench-agent") + + +class APIBase(ProcessWaiters, StatusCheckers): + """ + Base class with helper methods for Workbench API interactions. + Contains methods that handle the "how" of API operations. + """ + + def __init__(self, api_url: str, api_user: str, api_token: str): + """ + Initialize the base Workbench API client with authentication details. + + Args: + api_url: URL to the API endpoint + api_user: API username + api_token: API token/key + """ + # Ensure the API URL ends with api.php + if not api_url.endswith("/api.php"): + self.api_url = api_url.rstrip("/") + "/api.php" + logger.warning(f"API URL adjusted to: {self.api_url}") + else: + self.api_url = api_url + + self.api_user = api_user + self.api_token = api_token + self.session = requests.Session() # Use a session for potential connection reuse + self.session.trust_env = False # Do not trust .netrc file + + def _send_request(self, payload: dict, timeout: int = 1800) -> dict: + """ + Sends a POST request to the Workbench API with robust error handling. + + Args: + payload: The request payload + timeout: Request timeout in seconds + + Returns: + Dict with response data + + Raises: + NetworkError: For connection issues, timeouts, etc. + AuthenticationError: For authentication failures + ApiError: For API-level errors + ScanNotFoundError: When scan is not found + ProjectNotFoundError: When project is not found + ScanExistsError: When scan already exists + ProjectExistsError: When project already exists + """ + headers = { + "Accept": "*/*", + "Content-Type": "application/json; charset=utf-8", + } + + # Add authentication to payload + payload.setdefault("data", {}) + payload["data"]["username"] = self.api_user + payload["data"]["key"] = self.api_token + + req_body = json.dumps(payload) + logger.debug("API URL: %s", self.api_url) + logger.debug("Request Headers: %s", headers) + logger.debug("Request Body: %s", req_body) + + try: + response = self.session.post( + self.api_url, headers=headers, data=req_body, timeout=timeout + ) + logger.debug("Response Status Code: %s", response.status_code) + logger.debug("Response Text (first 500 chars): %s", response.text[:500]) + + # Handle authentication errors + if response.status_code == 401: + raise AuthenticationError("Invalid credentials or expired token") + + response.raise_for_status() # Raise HTTPError for bad responses (4xx or 5xx) + + try: + parsed_json = response.json() + + # Check for API-level errors indicated by status='0' + if isinstance(parsed_json, dict) and parsed_json.get("status") == "0": + error_msg = parsed_json.get("error", "Unknown API error") + logger.debug(f"API returned status 0: {error_msg} | Payload: {payload}") + + # Handle specific known errors + self._handle_api_errors(parsed_json, payload, error_msg) + + # If no specific error was handled, raise generic API error + raise ApiError(error_msg, code=parsed_json.get("code"), details=parsed_json) + + return parsed_json # Return successfully parsed JSON + + except json.JSONDecodeError as e: + logger.error( + f"Failed to decode JSON response: {response.text[:500]}", exc_info=True + ) + raise ApiError( + f"Invalid JSON received from API: {e.msg}", + details={"response_text": response.text[:500]}, + ) + + except requests.exceptions.ConnectionError as e: + logger.error("API connection failed: %s", e, exc_info=True) + raise NetworkError("Failed to connect to the API server", details={"error": str(e)}) + except requests.exceptions.Timeout as e: + logger.error("API request timed out: %s", e, exc_info=True) + raise NetworkError("Request to API server timed out", details={"error": str(e)}) + except requests.exceptions.RequestException as e: + logger.error("API request failed: %s", e, exc_info=True) + raise NetworkError(f"API request failed: {str(e)}", details={"error": str(e)}) + + def _handle_api_errors(self, parsed_json: dict, payload: dict, error_msg: str): + """ + Handle specific API errors and raise appropriate exceptions. + + Args: + parsed_json: The parsed JSON response + payload: The original request payload + error_msg: The error message from the API + """ + action = payload.get("action") + group = payload.get("group") + + # Handle existence check errors (non-fatal for existence checks) + is_existence_check = action == "get_information" + is_create_action = action == "create" + + # Project-specific errors + if group == "projects": + if is_existence_check and error_msg == "Project does not exist": + raise ProjectNotFoundError(f"Project not found") + elif is_create_action and "Project code already exists" in error_msg: + raise ProjectExistsError(f"Project already exists") + + # Scan-specific errors + elif group == "scans": + if is_existence_check and ( + "row_not_found" in error_msg or "Scan not found" in error_msg + ): + raise ScanNotFoundError(f"Scan not found") + elif is_create_action and ( + "Scan code already exists" in error_msg + or "Legacy.controller.scans.code_already_exists" in error_msg + ): + raise ScanExistsError(f"Scan already exists") diff --git a/src/workbench_agent/api/helpers/process_waiters.py b/src/workbench_agent/api/helpers/process_waiters.py new file mode 100644 index 0000000..cefa70c --- /dev/null +++ b/src/workbench_agent/api/helpers/process_waiters.py @@ -0,0 +1,474 @@ +import logging +import time +from typing import Dict, Any, Tuple, Callable, TYPE_CHECKING + +from ...exceptions import ( + ProcessTimeoutError, + ProcessError, + ApiError, + NetworkError, + ScanNotFoundError, +) + +logger = logging.getLogger("workbench-agent") + +class ProcessWaiters: + """ + Mixin class for waiting on long-running processes. + This class should be mixed into APIBase to provide waiting capabilities. + """ + + def _wait_for_process( + self, + process_description: str, + check_function: callable, + check_args: Dict[str, Any], + status_accessor: callable, + success_values: set, + failure_values: set, + max_tries: int, + wait_interval: int, + progress_indicator: bool = True + ): + """ + Generic process status checking and waiting function. + Repeatedly calls check_function until success, failure, or timeout. + + Args: + process_description: Human-readable description of the process being waited for + check_function: Function to call to check status + check_args: Arguments to pass to check_function + status_accessor: Function to extract status from check_function's result + success_values: Set of status values indicating success + failure_values: Set of status values indicating failure + max_tries: Maximum number of status checks before timeout + wait_interval: Seconds to wait between status checks + progress_indicator: Whether to print progress indicators (dots) + + Returns: + True if the process succeeded (status in success_values) + + Raises: + ProcessTimeoutError: If max_tries is reached before success/failure + ProcessError: If status is in failure_values + """ + logger.debug(f"Waiting for {process_description}...") + last_status = "UNKNOWN" + + for i in range(max_tries): + status_data = None + current_status = "UNKNOWN" + + try: + status_data = check_function(**check_args) + try: + current_status_raw = status_accessor(status_data) + current_status = str(current_status_raw).upper() + except Exception as access_err: + logger.warning(f"Error executing status_accessor during {process_description} check: {access_err}. Response data: {status_data}", exc_info=True) + current_status = "ACCESS_ERROR" # Treat as failure + + except Exception as e: + print() + print(f"Attempt {i+1}/{max_tries}: Error checking status for {process_description}: {e}") + print(f"Retrying in {wait_interval} seconds...") + logger.warning(f"Error calling check_function for {process_description}", exc_info=False) + time.sleep(wait_interval) + continue + + # Check for Success + if current_status in success_values: + print() + logger.debug(f"{process_description} completed successfully (Status: {current_status}).") + return True + + # Check for Failure (includes ACCESS_ERROR) + if current_status in failure_values or current_status == "ACCESS_ERROR": + print() # Newline after dots/status + base_error_msg = f"The {process_description} {current_status}" + error_detail = "" + if isinstance(status_data, dict): + error_detail = status_data.get("error", status_data.get("message", status_data.get("info", ""))) + if error_detail: + base_error_msg += f". Detail: {error_detail}" + raise ProcessError(base_error_msg, details=status_data) + + # Basic Status Printing + if current_status != last_status or i < 2 or i % 10 == 0: + print() + print(f"{process_description} status: {current_status}. Attempt {i+1}/{max_tries}.", end="", flush=True) + last_status = current_status + elif progress_indicator: + print(".", end="", flush=True) + + time.sleep(wait_interval) + + print() + raise ProcessTimeoutError( + f"Timeout waiting for {process_description} to complete after {max_tries * wait_interval} seconds (Last Status: {last_status}).", + details={"last_status": last_status, "max_tries": max_tries, "wait_interval": wait_interval, "last_data": status_data} + ) + + def wait_for_archive_extraction( + self, + scan_code: str, + scan_number_of_tries: int, + scan_wait_time: int, + ) -> Tuple[Dict[str, Any], float]: + """ + Wait for archive extraction to complete. + + Args: + scan_code: The code of the scan to check + scan_number_of_tries: Maximum number of attempts + scan_wait_time: Time to wait between attempts (ignored, fixed at 3 seconds) + + Returns: + Tuple[Dict[str, Any], float]: Tuple containing the final status data and the duration in seconds + + Raises: + ProcessTimeoutError: If the process times out + ProcessError: If the process fails + ApiError: If there are API issues + NetworkError: If there are network issues + """ + logger.debug(f"Waiting for archive extraction to complete for scan '{scan_code}'...") + + # Use fixed 3-second wait interval for archive extraction + archive_wait_interval = 3 + + last_status = "UNKNOWN" + last_state = "" + last_step = "" + client_start_time = time.time() # Start tracking client-side duration + status_data = {} + + for i in range(scan_number_of_tries): + try: + # Get current status from API using the mixin method + status_data = self.get_scan_status("EXTRACT_ARCHIVES", scan_code) + + # Extract key information + is_finished = str(status_data.get("is_finished", "0")) == "1" or status_data.get("is_finished") is True + current_status = status_data.get("status", "UNKNOWN").upper() + current_state = status_data.get("state", "") + percentage = status_data.get("percentage_done", "") + current_step = status_data.get("current_step", "") + current_file = status_data.get("current_filename", "") + info = status_data.get("info", "") + + # If finished flag is set, use FINISHED status + if is_finished: + current_status = "FINISHED" + + # Only print a new line when status, state, or step changes + details_changed = ( + current_status != last_status or + current_state != last_state or + current_step != last_step + ) + + # Print a new line on first status check + if i == 0: + details_changed = True + + # Check for success (finished) + if current_status == "FINISHED": + print("\nArchive Extraction completed successfully.") + logger.debug(f"Archive extraction for scan '{scan_code}' completed successfully") + # Calculate duration + duration = time.time() - client_start_time + # Add duration to status_data for reference + status_data["_duration_seconds"] = duration + return status_data, duration + + # Check for failure + if current_status in ["FAILED", "CANCELLED"]: + error_msg = f"Archive Extraction {current_status}" + if info: + error_msg += f" - Detail: {info}" + print(f"\n{error_msg}") + logger.error(f"Archive extraction for scan '{scan_code}' failed: {error_msg}") + raise ProcessError(error_msg, details=status_data) + + # Progress reporting + if details_changed: + print() + + # Construct a detailed status message + status_msg = f"Archive Extraction status: {current_status}" + if current_state: + status_msg += f" ({current_state})" + if percentage: + status_msg += f" - {percentage}" + if current_step: + status_msg += f" - Step: {current_step}" + if current_file: + status_msg += f" - File: {current_file}" + + print(f"{status_msg}. Attempt {i+1}/{scan_number_of_tries}", end="", flush=True) + + # Update last values + last_status = current_status + last_state = current_state + last_step = current_step + else: + # Just show a dot for minor updates + print(".", end="", flush=True) + + time.sleep(archive_wait_interval) + + except (ApiError, NetworkError, ScanNotFoundError) as e: + logger.error(f"Error checking archive extraction status for scan '{scan_code}': {e}", exc_info=True) + print(f"\nError checking Archive Extraction status: {e}") + raise + except ProcessError: + # Re-raise ProcessError directly + raise + except Exception as e: + logger.error(f"Unexpected error checking archive extraction status for scan '{scan_code}': {e}", exc_info=True) + print(f"\nUnexpected error during Archive Extraction status check: {e}") + raise ProcessError(f"Error during archive extraction for scan '{scan_code}'", details={"error": str(e)}) + + # If we exhaust all tries + logger.error(f"Timed out waiting for archive extraction to complete for scan '{scan_code}' after {scan_number_of_tries*archive_wait_interval} seconds") + print(f"\nTimed out waiting for Archive Extraction to complete") + raise ProcessTimeoutError( + f"Archive extraction timed out for scan '{scan_code}' after {scan_number_of_tries*archive_wait_interval} seconds", + details={"last_status": last_status, "max_tries": scan_number_of_tries, "wait_interval": archive_wait_interval} + ) + + def wait_for_scan_to_finish( + self, + scan_type: str, + scan_code: str, + scan_number_of_tries: int, + scan_wait_time: int, + ) -> Tuple[Dict[str, Any], float]: + """ + Wait for a scan to complete. Delegates to the consolidated implementation with appropriate parameters. + + Args: + scan_type: Type of scan ("SCAN" or "DEPENDENCY_ANALYSIS") + scan_code: Code of the scan to check + scan_number_of_tries: Maximum number of attempts + scan_wait_time: Time to wait between attempts + + Returns: + Tuple[Dict[str, Any], float]: Tuple containing the final status data and the duration in seconds + + Raises: + ProcessTimeoutError: If the process times out + ProcessError: If the process fails + ApiError: If there are API issues + NetworkError: If there are network issues + """ + if scan_type == "SCAN": + operation_name = "KB Scan" + should_track_files = True + elif scan_type == "DEPENDENCY_ANALYSIS": + operation_name = "Dependency Analysis" + should_track_files = False + elif scan_type == "REPORT_IMPORT": + operation_name = "SBOM Import" + should_track_files = False + else: + raise ValueError(f"Unsupported scan type: {scan_type}") + + return self._wait_for_operation_with_status( + operation_name=operation_name, + scan_type=scan_type, + scan_code=scan_code, + max_tries=scan_number_of_tries, + wait_interval=scan_wait_time, + should_track_files=should_track_files + ) + + def _wait_for_operation_with_status( + self, + operation_name: str, + scan_type: str, + scan_code: str, + max_tries: int, + wait_interval: int, + should_track_files: bool = False + ) -> Tuple[Dict[str, Any], float]: + """ + Consolidated implementation for waiting on scan operations with customized progress display. + + Args: + operation_name: Human-readable name of the operation (e.g., "KB Scan") + scan_type: API type of the scan ("SCAN" or "DEPENDENCY_ANALYSIS") + scan_code: Code of the scan to check + max_tries: Maximum number of attempts + wait_interval: Time to wait between attempts + should_track_files: Whether to track file counting information + + Returns: + Tuple[Dict[str, Any], float]: Tuple containing the final status data and the duration in seconds + + Raises: + ProcessTimeoutError: If the process times out + ProcessError: If the process fails + ApiError: If there are API issues + NetworkError: If there are network issues + """ + logger.debug(f"Waiting for {scan_type} operation to complete for scan '{scan_code}'...") + + # Initialize tracking variables + last_status = "UNKNOWN" + last_state = "" + last_step = "" + start_time = None + client_start_time = time.time() # Start tracking client-side duration + status_data = None + + for i in range(max_tries): + try: + # Get current status from API using the mixin method + status_data = self.get_scan_status(scan_type, scan_code) + + # Extract key information + current_status = status_data.get("status", "UNKNOWN").upper() + current_state = status_data.get("state", "") + current_step = status_data.get("current_step", "") + + # Extract additional information specific to the scan type + file_count_info = "" + total_files = 0 + current_file_idx = 0 + percentage = "" + + if should_track_files: + # Extract file processing information (KB scan only) + total_files = status_data.get("total_files", 0) + current_file_idx = status_data.get("current_file", 0) + percentage = status_data.get("percentage_done", "") + + # Create file progress info if available + if total_files and int(total_files) > 0: + # Display progress as a fraction with percentage + file_count_info = f" - File {current_file_idx}/{total_files}" + if percentage: + file_count_info += f" ({percentage})" + + # Get operation start time from API if available + api_start_time = status_data.get("started") + if api_start_time and not start_time: + start_time = api_start_time + + # Only print a new line when status, state, or step changes + details_changed = ( + current_status != last_status or + current_state != last_state or + current_step != last_step + ) + + # Print a new line on first status check + if i == 0: + details_changed = True + + # Print a new line every 10 status checks to update the user + show_periodic_update = i > 0 and i % 10 == 0 and current_status == "RUNNING" + + # Check for success (finished) + if current_status == "FINISHED" or status_data.get("is_finished") in [True, "1", 1]: + # Calculate duration using API timestamps if available + duration_str = "" + api_finish_time = status_data.get("finished") + api_duration_sec = None + + if start_time and api_finish_time: + try: + # Parse timestamps and calculate duration + from datetime import datetime + start_dt = datetime.strptime(start_time, "%Y-%m-%d %H:%M:%S") + finish_dt = datetime.strptime(api_finish_time, "%Y-%m-%d %H:%M:%S") + api_duration_sec = (finish_dt - start_dt).total_seconds() + + # Format duration as a string + minutes, seconds = divmod(api_duration_sec, 60) + hours, minutes = divmod(minutes, 60) + if hours > 0: + duration_str = f" (Completed in {int(hours)}h {int(minutes)}m {int(seconds)}s)" + elif minutes > 0: + duration_str = f" (Completed in {int(minutes)}m {int(seconds)}s)" + else: + duration_str = f" (Completed in {int(seconds)}s)" + except Exception as e: + logger.debug(f"Error calculating duration: {e}") + + print(f"\n{operation_name} completed successfully{duration_str}.") + logger.debug(f"{scan_type} for scan '{scan_code}' completed successfully") + + # Calculate client-side duration as fallback + client_duration = time.time() - client_start_time + + # Prefer API-reported duration if available, otherwise use client-side duration + final_duration = api_duration_sec if api_duration_sec is not None else client_duration + + # Add duration to status_data for reference + status_data["_duration_seconds"] = final_duration + + return status_data, final_duration + + # Check for failure + if current_status in ["FAILED", "CANCELLED"]: + error_msg = f"{operation_name} {current_status}" + info = status_data.get("info", "") + if info: + error_msg += f" - Detail: {info}" + print(f"\n{error_msg}") + logger.error(f"{scan_type} for scan '{scan_code}' failed: {error_msg}") + raise ProcessError(error_msg, details=status_data) + + # Progress reporting + if details_changed or show_periodic_update: + print() + + # Construct a detailed status message + status_msg = f"{operation_name} status: {current_status}" + if current_state: + status_msg += f" ({current_state})" + + # Include file progress information for KB scan + if file_count_info: + status_msg += file_count_info + elif percentage: + status_msg += f" - {percentage}" + + # Show current step + if current_step: + status_msg += f" - Step: {current_step}" + + print(f"{status_msg}. Attempt {i+1}/{max_tries}", end="", flush=True) + + # Update last values + last_status = current_status + last_state = current_state + last_step = current_step + else: + # Just show a dot for minor updates + print(".", end="", flush=True) + + time.sleep(wait_interval) + + except (ApiError, NetworkError, ScanNotFoundError) as e: + logger.error(f"Error checking {scan_type} status for scan '{scan_code}': {e}", exc_info=True) + print(f"\nError checking {operation_name} status: {e}") + raise + except ProcessError: + # Re-raise ProcessError directly + raise + except Exception as e: + logger.error(f"Unexpected error checking {scan_type} status for scan '{scan_code}': {e}", exc_info=True) + print(f"\nUnexpected error during {operation_name} status check: {e}") + raise ProcessError(f"Error during {scan_type} operation for scan '{scan_code}'", details={"error": str(e)}) + + # If we exhaust all tries + logger.error(f"Timed out waiting for {scan_type} to complete for scan '{scan_code}' after {max_tries} attempts") + print(f"\nTimed out waiting for {operation_name} to complete") + raise ProcessTimeoutError( + f"{scan_type} timed out for scan '{scan_code}' after {max_tries} attempts", + details={"last_status": last_status, "max_tries": max_tries, "wait_interval": wait_interval} + ) diff --git a/src/workbench_agent/api/helpers/project_scan_resolvers.py b/src/workbench_agent/api/helpers/project_scan_resolvers.py new file mode 100644 index 0000000..12a2352 --- /dev/null +++ b/src/workbench_agent/api/helpers/project_scan_resolvers.py @@ -0,0 +1,182 @@ +from typing import Dict, List, Optional, Union, Any, Tuple +import logging +import time +import argparse +from ..projects_api import ProjectsAPI +from ..scans_api import ScansAPI +from ...exceptions import ( + WorkbenchAgentError, + ApiError, + NetworkError, + ConfigurationError, + ValidationError, + ProjectNotFoundError, + ScanNotFoundError, + ProjectExistsError, + ScanExistsError +) + +# Assume logger is configured in main.py +logger = logging.getLogger("workbench-agent") + + +class ResolveWorkbenchProjectScan(ProjectsAPI, ScansAPI): + """ + Workbench API Scan Target Resolution Operations - handles resolving project names to codes + and scan names to codes/IDs, with optional creation functionality. + + Inherits from both ProjectsAPI and ScansAPI to access list methods. + """ + + def resolve_project(self, project_name: str, create_if_missing: bool = False) -> str: + """Find a project by name, optionally creating it if not found.""" + # Look for existing project + projects = self.list_projects() + project = next((p for p in projects if p.get("project_name") == project_name), None) + + if project: + return project["project_code"] + + # Create if requested + if create_if_missing: + print(f"Creating project '{project_name}'...") + try: + self.create_project(project_name) + return project_name # Use project_name as project_code + except ProjectExistsError: + # Handle race condition + projects = self.list_projects() + project = next((p for p in projects if p.get("project_name") == project_name), None) + if project: + return project["project_code"] + raise ApiError(f"Failed to resolve project '{project_name}' after creation conflict") + + raise ProjectNotFoundError(f"Project '{project_name}' not found") + + def resolve_scan(self, scan_name: str, project_name: Optional[str], create_if_missing: bool, params: argparse.Namespace, import_from_report: bool = False) -> Tuple[str, int]: + """Find a scan by name, optionally creating it if not found.""" + if project_name: + # Look in specific project + project_code = self.resolve_project(project_name, create_if_missing) + scan_list = self.get_project_scans(project_code) + + # Look for exact match only + scan = next((s for s in scan_list if s.get('name') == scan_name), None) + if scan: + return scan['code'], int(scan['id']) + + # Create if requested + if create_if_missing: + print(f"Creating scan '{scan_name}' in project '{project_name}'...") + git_params = self._get_git_params(params) + scan_id = self.create_webapp_scan( + scan_code=scan_name, # Use scan_name as scan_code + project_code=project_code, + **git_params + ) + time.sleep(2) # Brief wait for creation to process + + # Get the newly created scan + scan_list = self.get_project_scans(project_code) + scan = next((s for s in scan_list if s.get('name') == scan_name), None) + if scan: + return scan['code'], int(scan['id']) + raise ApiError(f"Failed to retrieve newly created scan '{scan_name}'") + + raise ScanNotFoundError(f"Scan '{scan_name}' not found in project '{project_name}'") + + else: + # Global search + if create_if_missing: + raise ConfigurationError("Cannot create a scan without specifying a project") + + scan_list = self.list_scans() + found = [s for s in scan_list if s.get('name') == scan_name] + + if len(found) == 1: + scan = found[0] + return scan['code'], int(scan['id']) + elif len(found) > 1: + projects = sorted(set(s.get('project_code', 'Unknown') for s in found)) + raise ValidationError(f"Multiple scans found with name '{scan_name}' in projects: {', '.join(projects)}") + + raise ScanNotFoundError(f"Scan '{scan_name}' not found in any project") + + def prepare_project_and_scan(self, project_identifier: str, scan_identifier: str, params: argparse.Namespace = None) -> Tuple[str, int]: + """ + Ensures project exists and creates scan if needed. + Supports both legacy code-based approach and new name-based approach with automatic resolution. + + Args: + project_identifier: Project code/name to use + scan_identifier: Scan code/name to use + params: Optional command line parameters for name resolution + + Returns: + Tuple of (project_code, scan_id) + """ + # Determine if we're using names vs codes based on explicit flag + use_name_resolution = params and getattr(params, 'use_name_resolution', False) + + if use_name_resolution: + logger.info(f"Using name-based resolution for project and scan...") + + # Get names from params + project_name = getattr(params, 'project_name', project_identifier) + scan_name = getattr(params, 'scan_name', scan_identifier) + + logger.info(f"Resolving project name '{project_name}' and scan name '{scan_name}'...") + + # Resolve project (auto-create if missing) + try: + resolved_project_code = self.resolve_project(project_name, create_if_missing=True) + logger.info(f"Project '{project_name}' resolved to code '{resolved_project_code}'") + except Exception as e: + logger.error(f"Failed to resolve project '{project_name}': {e}") + raise + + # Resolve scan (auto-create if missing) + try: + resolved_scan_code, scan_id = self.resolve_scan( + scan_name=scan_name, + project_name=project_name, + create_if_missing=True, + params=params + ) + logger.info(f"Scan '{scan_name}' resolved to code '{resolved_scan_code}' with ID {scan_id}") + return resolved_project_code, scan_id + except Exception as e: + logger.error(f"Failed to resolve scan '{scan_name}': {e}") + raise + + else: + # Legacy code-based approach + logger.info(f"Using legacy code-based approach for project '{project_identifier}' and scan '{scan_identifier}'...") + + # Try to create project (more efficient than check + create) + try: + self.create_project(project_identifier) + logger.info(f"Project '{project_identifier}' created successfully.") + except (ProjectExistsError, ApiError, NetworkError) as e: + logger.info(f"Project '{project_identifier}' may already exist or creation failed: {e}") + # Continue processing - the project might already exist + + # Try to create scan (more efficient than check + create) + try: + scan_id = self.create_webapp_scan( + scan_code=scan_identifier, + project_code=project_identifier + ) + logger.info(f"Created scan '{scan_identifier}' with ID: {scan_id}") + except (ScanExistsError, ApiError, NetworkError) as e: + logger.info(f"Scan '{scan_identifier}' may already exist or creation failed: {e}") + # For existing scans, we don't need the scan_id for our workflows + scan_id = None + + return project_identifier, scan_id + + def _get_git_params(self, params: argparse.Namespace) -> Dict[str, Any]: + """Get git parameters if this is a git scan.""" + # For now, the workbench-agent doesn't support git-specific scans like the CLI + # This is a placeholder for future git scan support + return {} \ No newline at end of file diff --git a/src/workbench_agent/api/helpers/status_checkers.py b/src/workbench_agent/api/helpers/status_checkers.py new file mode 100644 index 0000000..4df66ea --- /dev/null +++ b/src/workbench_agent/api/helpers/status_checkers.py @@ -0,0 +1,281 @@ +import logging +from typing import Dict, Any, List, TYPE_CHECKING + +from ...exceptions import ( + ApiError, + NetworkError, + CompatibilityError, + ProcessError, + ProcessTimeoutError, + ScanNotFoundError, +) + +logger = logging.getLogger("workbench-agent") + +class StatusCheckers: + """ + Mixin class for checking process statuses. + This class should be mixed into APIBase to provide status checking capabilities. + """ + + def _is_status_check_supported(self, scan_code: str, process_type: str) -> bool: + """ + Checks if the Workbench instance likely supports check_status for a given process type + by probing the API and analyzing the response, including specific error codes. + + Args: + scan_code: The code of the scan to check against. + process_type: The process type string (e.g., "EXTRACT_ARCHIVES"). + + Returns: + True if the check_status call for the type seems supported, False otherwise. + + Raises: + ApiError: If the check_status call fails for reasons other than a recognized unsupported type error. + NetworkError: If there are network connectivity issues. + """ + logger.debug(f"Probing check_status support for type '{process_type}' on scan '{scan_code}'...") + payload = { + "group": "scans", + "action": "check_status", + "data": { + "scan_code": scan_code, + "type": process_type.upper(), + }, + } + try: + # Short timeout is sufficient for the probe. + response = self._send_request(payload, timeout=30) + + # If status is "1", the API understood the request type. + if response.get("status") == "1": + logger.debug(f"check_status for type '{process_type}' appears to be supported (API status 1).") + return True + + # --- Check for specific 'invalid type' error structure --- + elif response.get("status") == "0": + error_code = response.get("error") + data_list = response.get("data") + + # Check for the specific error structure indicating an invalid 'type' option + if (error_code == "RequestData.Base.issues_while_parsing_request" and + isinstance(data_list, list) and len(data_list) > 0 and + isinstance(data_list[0], dict) and + data_list[0].get("code") == "RequestData.Base.field_not_valid_option" and + data_list[0].get("message_parameters", {}).get("fieldname") == "type"): + + logger.warning(f"This version of Workbench does not support check_status for '{process_type}'. ") + + # Optionally log the valid types listed by the API + valid_options = data_list[0].get("message_parameters", {}).get("options") + if valid_options: + logger.debug(f"API reported valid types are: [{valid_options}]") + return False + else: + # It's a different status 0 error (e.g., scan not found), raise it. + logger.error(f"API error during {process_type} support check (but not an invalid type error): {error_code} - {response.get('message')}") + raise ApiError(f"API error during {process_type} support check: {error_code} - {response.get('message', 'No details')}", details=response) + + else: + # Unexpected response format (neither status 1 nor 0) + logger.warning(f"Unexpected response format during {process_type} support check: {response}") + # Assume not supported to be safe + return False + + except Exception as e: + # This block now primarily catches network errors or unexpected exceptions from _send_request. + # We add a fallback check on the exception message just in case _send_request's logic changes. + error_msg_lower = str(e).lower() + if "requestdata.base.field_not_valid_option" in error_msg_lower and "type" in error_msg_lower: + logger.warning( + f"Workbench likely does not support check_status for type '{process_type}'. " + f"Skipping status check. (Detected via exception: {e})" + ) + return False + else: + # Different error (network, scan not found, etc.), re-raise it. + logger.error(f"Unexpected exception during {process_type} support check: {e}", exc_info=False) + if isinstance(e, NetworkError): + raise + raise ApiError(f"Unexpected error during {process_type} support check", details={"error": str(e)}) from e + + def _standard_scan_status_accessor(self, data: Dict[str, Any]) -> str: + """ + Standard status accessor for extracting status from API responses. + Works with responses from SCAN, DEPENDENCY_ANALYSIS, EXTRACT_ARCHIVES and other operations. + + This method handles various status formats and normalizes them: + 1. Checks if 'is_finished' flag indicates completion (returns "FINISHED") + 2. Falls back to the 'status' field if present + 3. Returns "UNKNOWN" if neither is available + 4. Handles errors gracefully by returning "ACCESS_ERROR" + + Args: + data: Response data dictionary from an API call + + Returns: + str: Normalized uppercase status string ("FINISHED", "RUNNING", "QUEUED", "FAILED", etc.) + """ + try: + # Some API endpoints use is_finished=1/true to indicate completion + is_finished_flag = data.get("is_finished") + is_finished = str(is_finished_flag) == "1" or is_finished_flag is True + + # If finished, return "FINISHED" (using the hardcoded success value) + if is_finished: + return "FINISHED" + + # Otherwise, return the value of the 'status' key (or UNKNOWN) + # Make sure it's uppercase for consistent comparison + status = data.get("status", "UNKNOWN") + if status: + return status.upper() + return "UNKNOWN" + except (ValueError, TypeError, AttributeError) as e: + logger.warning(f"Error accessing status keys in data: {data}", exc_info=True) + return "ACCESS_ERROR" # Use the ACCESS_ERROR state + + def check_status(self, scan_type: str, scan_code: str) -> Dict[str, Any]: + """ + Calls API scans -> check_status to determine if the process is finished. + + Args: + scan_type: One of these: SCAN, DEPENDENCY_ANALYSIS + scan_code: The unique identifier for the scan + + Returns: + dict: The data section from the JSON response returned from API + + Raises: + ApiError: If the API call fails + ScanNotFoundError: If the scan doesn't exist + """ + logger.debug(f"Checking status for {scan_type} on scan '{scan_code}'") + + payload = { + "group": "scans", + "action": "check_status", + "data": { + "scan_code": scan_code, + "type": scan_type, + }, + } + + response = self._send_request(payload) + if response.get("status") == "1" and "data" in response: + return response["data"] + else: + error_msg = response.get("error", "Unknown error") + if "Scan not found" in error_msg or "row_not_found" in error_msg: + raise ScanNotFoundError(f"Scan '{scan_code}' not found") + raise ApiError( + f"Failed to check status for {scan_type} on scan '{scan_code}': {error_msg}", + details=response, + ) + + def get_scan_status(self, scan_type: str, scan_code: str) -> dict: + """ + Retrieve scan status. + + Args: + scan_type: Type of scan operation (SCAN, DEPENDENCY_ANALYSIS, or EXTRACT_ARCHIVES) + scan_code: Code of the scan to check + + Returns: + dict: The scan status data + + Raises: + ApiError: If there are API issues + ScanNotFoundError: If the scan doesn't exist + NetworkError: If there are network issues + """ + return self.check_status(scan_type, scan_code) + + def ensure_scan_is_idle( + self, + scan_code: str, + process_types_to_check: List[str], + scan_number_of_tries: int = 10, + scan_wait_time: int = 30 + ): + """ + Ensures specified background processes for a scan are idle (not RUNNING or QUEUED). + If a process is running/queued, waits for it to finish before proceeding. + + This method can handle multiple process types at once and supports various process types + including SCAN, DEPENDENCY_ANALYSIS, GIT_CLONE, EXTRACT_ARCHIVES, and REPORT_IMPORT. + + Args: + scan_code: Code of the scan to check + process_types_to_check: List of process types to check (e.g., ["SCAN", "DEPENDENCY_ANALYSIS"]) + scan_number_of_tries: Maximum number of attempts for waiting + scan_wait_time: Time to wait between attempts + + Raises: + ProcessError: If there are process-related issues + ApiError: If there are API issues + NetworkError: If there are network issues + """ + logger.debug(f"Asserting idle status for processes {process_types_to_check} on scan '{scan_code}'...") + while True: + all_processes_idle_this_pass = True + logger.debug("Starting a new pass to check idle status...") + for process_type in process_types_to_check: + process_type_upper = process_type.upper() + logger.debug(f"Checking status for process type: {process_type_upper}") + current_status = "UNKNOWN" + try: + if process_type_upper in ["SCAN", "DEPENDENCY_ANALYSIS", "REPORT_IMPORT"]: + status_data = self.get_scan_status(process_type_upper, scan_code) + current_status = status_data.get("status", "UNKNOWN").upper() + elif process_type_upper == "EXTRACT_ARCHIVES": + # EXTRACT_ARCHIVES status checking is handled differently + # Check if status checking is supported for this process type + if self._is_status_check_supported(scan_code, "EXTRACT_ARCHIVES"): + # Use the specialized method for checking archive extraction status + try: + status_data = self.get_scan_status("EXTRACT_ARCHIVES", scan_code) + current_status = self._standard_scan_status_accessor(status_data) + except (ApiError, ScanNotFoundError) as e: + logger.debug(f"Could not check EXTRACT_ARCHIVES status, assuming finished: {e}") + current_status = "FINISHED" + else: + logger.debug(f"EXTRACT_ARCHIVES status checking not supported. Assuming idle.") + current_status = "FINISHED" + else: + logger.warning(f"Unknown process type '{process_type_upper}' requested for idle check. Skipping.") + continue + logger.debug(f"Current status for {process_type_upper}: {current_status}") + except ScanNotFoundError: + logger.debug(f"Scan '{scan_code}' not found during idle check for {process_type_upper}. Assuming idle.") + print(f" - {process_type_upper}: Not found (considered idle).") + continue + except (ApiError, NetworkError) as e: + raise ProcessError(f"Cannot proceed: Failed to check status for {process_type_upper} due to API/Network error: {e}") from e + except Exception as e: + raise ProcessError(f"Cannot proceed: Unexpected error checking status for {process_type_upper}: {e}") from e + + if current_status in ["RUNNING", "QUEUED", "NOT FINISHED"]: + all_processes_idle_this_pass = False + print(f" - {process_type_upper}: Status is {current_status}. Waiting for completion...") + try: + # Use the mixin methods directly since we're now a mixin + if process_type_upper == "EXTRACT_ARCHIVES": + # Use the specialized wait method with 3-second intervals for archive extraction + _, _ = self.wait_for_archive_extraction(scan_code, scan_number_of_tries, scan_wait_time) + else: + _, _ = self.wait_for_scan_to_finish(process_type_upper, scan_code, scan_number_of_tries, scan_wait_time) + print(f" - {process_type_upper}: Previous run finished.") + logger.debug(f"Breaking inner loop after waiting for {process_type_upper} to re-check all statuses.") + break + except (ProcessTimeoutError, ProcessError) as wait_err: + raise ProcessError(f"Cannot proceed: Waiting for existing {process_type_upper} failed: {wait_err}") from wait_err + except Exception as wait_exc: + raise ProcessError(f"Cannot proceed: Unexpected error waiting for {process_type_upper}: {wait_exc}") from wait_exc + else: + print(f" - {process_type_upper}: Status is {current_status} (considered idle).") + + if all_processes_idle_this_pass: + logger.debug("All processes confirmed idle in this pass. Exiting check loop.") + break + print("All Scan processes confirmed idle! Proceeding...") diff --git a/src/workbench_agent/api/helpers/upload_helpers.py b/src/workbench_agent/api/helpers/upload_helpers.py new file mode 100644 index 0000000..6ecc9a1 --- /dev/null +++ b/src/workbench_agent/api/helpers/upload_helpers.py @@ -0,0 +1,314 @@ +from typing import Generator +import io +import logging +import json +import requests +import time +import os + +from .api_base import APIBase +from ...exceptions import NetworkError, ApiError, FileSystemError + +logger = logging.getLogger("workbench-agent") + + +class UploadHelper(APIBase): + """ + Helper mixin for handling chunked file uploads and progress tracking. + This class should be mixed into API classes to provide upload capabilities. + """ + + # Upload Constants + CHUNKED_UPLOAD_THRESHOLD = 16 * 1024 * 1024 # 16MB + CHUNK_SIZE = 5 * 1024 * 1024 # 5MB + MAX_CHUNK_RETRIES = 3 + PROGRESS_UPDATE_INTERVAL = 20 # Percent + SMALL_FILE_CHUNK_THRESHOLD = 5 # Always show progress for ≤5 chunks + + def _read_in_chunks( + self, file_object: io.BufferedReader, chunk_size: int = 5 * 1024 * 1024 + ) -> Generator[bytes, None, None]: + """ + Generator to read a file piece by piece. + + Args: + file_object: The payload of the request + chunk_size: Size of the chunk. Default chunk size is 5MB + """ + while True: + data = file_object.read(chunk_size) + if not data: + break + yield data + + def _upload_single_chunk(self, chunk: bytes, chunk_number: int, headers: dict) -> None: + """ + Upload a single chunk with retry logic. + + Args: + chunk: The chunk data to upload + chunk_number: The chunk number (for logging) + headers: Headers for the upload request + + Raises: + NetworkError: If there are network issues after all retries + ApiError: If the upload fails after all retries + """ + retry_count = 0 + + while retry_count <= self.MAX_CHUNK_RETRIES: + try: + # Create request manually to remove Content-Length header + req = requests.Request( + "POST", + self.api_url, + headers=headers, + data=chunk, + auth=(self.api_user, self.api_token), + ) + + # Create a fresh session for each chunk + chunk_session = requests.Session() + prepped = chunk_session.prepare_request(req) + if "Content-Length" in prepped.headers: + del prepped.headers["Content-Length"] + logger.debug(f"Removed Content-Length header for chunk {chunk_number}") + + # Send the request + resp_chunk = chunk_session.send(prepped, timeout=1800) + + # Validate response + self._validate_chunk_response(resp_chunk, chunk_number, retry_count) + return # Success! + + except requests.exceptions.RequestException as e: + if retry_count < self.MAX_CHUNK_RETRIES: + logger.warning( + f"Chunk {chunk_number} network error (attempt {retry_count + 1}/{self.MAX_CHUNK_RETRIES + 1}): {e}" + ) + retry_count += 1 + time.sleep(2) # Longer delay for network issues + continue + else: + logger.error( + f"Chunk {chunk_number} failed after {self.MAX_CHUNK_RETRIES + 1} attempts: {e}" + ) + raise NetworkError( + f"Network error for chunk {chunk_number} after {self.MAX_CHUNK_RETRIES + 1} attempts: {e}" + ) + + def _validate_chunk_response( + self, response: requests.Response, chunk_number: int, retry_count: int + ) -> None: + """ + Validate chunk upload response and handle retries. + + Args: + response: The HTTP response + chunk_number: The chunk number (for logging) + retry_count: Current retry attempt + + Raises: + requests.exceptions.RequestException: For retryable errors + NetworkError: For non-retryable errors + """ + # Check HTTP status + if response.status_code != 200: + error_msg = f"HTTP {response.status_code}: {response.text[:200]}" + if retry_count < self.MAX_CHUNK_RETRIES: + logger.warning( + f"Chunk {chunk_number} failed (attempt {retry_count + 1}/{self.MAX_CHUNK_RETRIES + 1}): {error_msg}" + ) + time.sleep(1) + raise requests.exceptions.RequestException(f"HTTP {response.status_code}") + else: + logger.error( + f"Chunk {chunk_number} upload failed after {self.MAX_CHUNK_RETRIES + 1} attempts: {error_msg}" + ) + response.raise_for_status() + + # Validate JSON response + try: + response.json() + logger.debug(f"Chunk {chunk_number} response JSON parsed successfully") + except json.JSONDecodeError: + error_msg = f"Invalid JSON response: {response.text[:200]}" + if retry_count < self.MAX_CHUNK_RETRIES: + logger.warning( + f"Chunk {chunk_number} JSON parsing failed (attempt {retry_count + 1}/{self.MAX_CHUNK_RETRIES + 1}): {error_msg}" + ) + time.sleep(1) + raise requests.exceptions.RequestException("JSON decode error") + else: + logger.error( + f"Chunk {chunk_number} upload: Failed to decode JSON response after {self.MAX_CHUNK_RETRIES + 1} attempts" + ) + raise NetworkError( + f"Invalid JSON response from server for chunk {chunk_number}: {error_msg}" + ) + + def _should_show_progress( + self, progress_percent: int, last_progress: int, chunk_number: int, total_chunks: int + ) -> bool: + """ + Determine if progress should be displayed. + """ + return ( + progress_percent >= last_progress + self.PROGRESS_UPDATE_INTERVAL + or total_chunks <= self.SMALL_FILE_CHUNK_THRESHOLD + or chunk_number == total_chunks + ) + + def _format_progress_display( + self, + progress_percent: int, + chunk_number: int, + total_chunks: int, + bytes_uploaded: int, + elapsed_time: float, + ) -> str: + """ + Format progress display string with performance metrics. + """ + # Calculate speed + speed_mbps = (bytes_uploaded / (1024 * 1024)) / elapsed_time + if speed_mbps >= 1: + speed_str = f"{speed_mbps:.1f}MB/s" + else: + speed_str = f"{speed_mbps * 1024:.0f}KB/s" + + # Calculate ETA + if bytes_uploaded > 0 and hasattr(self, "_total_file_size"): + remaining_bytes = self._total_file_size - bytes_uploaded + eta_seconds = remaining_bytes / (bytes_uploaded / elapsed_time) + if eta_seconds > 60: + eta_str = f"ETA ~{eta_seconds/60:.0f}m" + else: + eta_str = f"ETA ~{eta_seconds:.0f}s" + else: + eta_str = "ETA ~?s" + + return f"Upload progress: {progress_percent:3d}% ({chunk_number}/{total_chunks} chunks) - {speed_str} - {eta_str}" + + def _perform_upload(self, file_path: str, headers: dict) -> None: + """ + Performs the upload of a single file, using chunking if necessary. + + Args: + file_path: Path to the file to upload + headers: The pre-constructed headers for the upload request + + Raises: + FileSystemError: If there are file system errors + NetworkError: If there are network issues + ApiError: If the upload fails + """ + file_handle = None + try: + file_size = os.path.getsize(file_path) + upload_basename = os.path.basename(file_path) + + logger.info(f"Uploading file '{upload_basename}' ({file_size} bytes)...") + logger.debug(f"Upload Request Headers: {headers}") + + file_handle = open(file_path, "rb") + + if file_size > self.CHUNKED_UPLOAD_THRESHOLD: + logger.info( + f"File size exceeds threshold ({self.CHUNKED_UPLOAD_THRESHOLD} bytes). Using chunked upload..." + ) + + # Add chunked upload headers + headers_copy = headers.copy() + headers_copy["Transfer-Encoding"] = "chunked" + headers_copy["Content-Type"] = "application/octet-stream" + + total_chunks = (file_size + self.CHUNK_SIZE - 1) // self.CHUNK_SIZE + bytes_uploaded = 0 + start_time = time.time() + last_progress_print = 0 + + self._total_file_size = file_size + + print( + f"Uploading {file_size / (1024*1024):.1f}MB in {total_chunks} {self.CHUNK_SIZE / (1024*1024):.0f}MB chunks." + ) + + for i, chunk in enumerate( + self._read_in_chunks(file_handle, chunk_size=self.CHUNK_SIZE) + ): + chunk_number = i + 1 + bytes_uploaded += len(chunk) + + self._upload_single_chunk(chunk, chunk_number, headers_copy) + + progress_percent = min(100, (bytes_uploaded * 100) // file_size) + elapsed_time = time.time() - start_time + + if ( + self._should_show_progress( + progress_percent, last_progress_print, chunk_number, total_chunks + ) + and elapsed_time > 0 + ): + progress_message = self._format_progress_display( + progress_percent, + chunk_number, + total_chunks, + bytes_uploaded, + elapsed_time, + ) + print(progress_message) + last_progress_print = progress_percent + + elapsed_time = time.time() - start_time + avg_speed = ( + (bytes_uploaded / (1024 * 1024)) / elapsed_time if elapsed_time > 0 else 0 + ) + print( + f"Chunked upload completed! {bytes_uploaded / (1024*1024):.1f}MB uploaded in {elapsed_time:.1f}s (avg: {avg_speed:.1f}MB/s)" + ) + + if hasattr(self, "_total_file_size"): + delattr(self, "_total_file_size") + else: + # Standard upload for smaller files + logger.info( + f"Using regular upload for file '{upload_basename}' (size: {file_size} bytes)" + ) + + resp = self.session.post( + self.api_url, + headers=headers, + data=file_handle, + auth=(self.api_user, self.api_token), + timeout=1800, + ) + + logger.debug(f"HTTP Status Code: {resp.status_code}") + + if resp.status_code != 200: + error_msg = f"Request failed with status code {resp.status_code}" + reason = resp.reason + response_text = resp.text + logger.error(f"{error_msg}, Reason: {reason}, Response: {response_text}") + raise ApiError(f"Upload failed: {error_msg} - {reason}") + + # Validate JSON response + try: + resp.json() + except json.JSONDecodeError: + logger.error(f"Failed to decode JSON: {resp.text}") + raise ApiError(f"Invalid JSON response from upload API: {resp.text[:500]}") + + logger.info(f"Finished uploading '{upload_basename}' using regular upload") + + except IOError as e: + logger.error(f"Failed to upload file '{file_path}': {e}", exc_info=True) + raise FileSystemError(f"File I/O error during upload: {e}") + except requests.exceptions.RequestException as e: + logger.error(f"Network error during upload: {e}", exc_info=True) + raise NetworkError(f"Network error during upload: {e}") + finally: + if file_handle and not file_handle.closed: + file_handle.close() diff --git a/src/workbench_agent/api/projects_api.py b/src/workbench_agent/api/projects_api.py new file mode 100644 index 0000000..f221af2 --- /dev/null +++ b/src/workbench_agent/api/projects_api.py @@ -0,0 +1,253 @@ +import logging +from typing import Dict, Any, List, Optional +from .helpers.api_base import APIBase +from ..exceptions import ApiError, ProjectNotFoundError, ProjectExistsError, ValidationError + +logger = logging.getLogger("workbench-agent") + + +class ProjectsAPI(APIBase): + """ + Workbench API Project Operations. + """ + + # --- Enhanced Validation Methods --- + + def _validate_project_parameters(self, project_code: str) -> None: + """ + Validates project parameters before API operations. + + Args: + project_code: The project code to validate + + Raises: + ValidationError: If parameters are invalid + """ + if not project_code or not project_code.strip(): + raise ValidationError("Project code cannot be empty") + + # --- Project Information Methods --- + + def get_project_information(self, project_code: str) -> Dict[str, Any]: + """ + Retrieves detailed information about a project. + + Args: + project_code: Code of the project to get information for + + Returns: + Dict containing project information + + Raises: + ProjectNotFoundError: If the project doesn't exist + ApiError: If there are API issues + NetworkError: If there are network issues + """ + logger.debug(f"Getting project information for '{project_code}'") + + payload = { + "group": "projects", + "action": "get_information", + "data": { + "project_code": project_code, + }, + } + + response = self._send_request(payload) + if response.get("status") == "1" and "data" in response: + return response["data"] + else: + error_msg = response.get("error", "Unknown error") + if "Project does not exist" in error_msg or "row_not_found" in error_msg: + raise ProjectNotFoundError(f"Project '{project_code}' not found") + raise ApiError( + f"Failed to get project information for '{project_code}': {error_msg}", + details=response, + ) + + def check_if_project_exists(self, project_code: str) -> bool: + """ + Check if project exists (backwards compatibility with original agent). + + Args: + project_code: The unique identifier for the project + + Returns: + bool: True if project exists, False otherwise + """ + try: + self.get_project_information(project_code) + return True + except ProjectNotFoundError: + return False + except (ApiError, Exception): + # On other errors, assume project doesn't exist for safety + return False + + # --- Existing Methods with Enhanced Organization --- + + def list_projects(self) -> List[Dict[str, Any]]: + """ + List all projects accessible to the current user. + + Returns: + List[Dict]: List of project dictionaries with keys like project_code, project_name, etc. + + Raises: + ApiError: If the API call fails + NetworkError: If there are network issues + """ + logger.debug("Listing all projects") + + payload = { + "group": "projects", + "action": "get_all_projects", + "data": {} + } + + response = self._send_request(payload) + if response.get("status") == "1" and "data" in response: + projects = response["data"] + if isinstance(projects, list): + logger.debug(f"Found {len(projects)} projects") + return projects + elif isinstance(projects, dict): + # Sometimes API returns dict instead of list + logger.debug(f"Found {len(projects)} projects (as dict)") + return list(projects.values()) if projects else [] + else: + logger.warning(f"Expected list or dict for projects, got {type(projects)}") + return [] + else: + error_msg = response.get("error", f"Unexpected response: {response}") + raise ApiError(f"Failed to list projects: {error_msg}", details=response) + + def get_project_scans(self, project_code: str) -> List[Dict[str, Any]]: + """ + Get all scans for a specific project. + + Args: + project_code: The project code to get scans for + + Returns: + List[Dict]: List of scan dictionaries for the specified project + + Raises: + ApiError: If the API call fails + NetworkError: If there are network issues + """ + logger.debug(f"Getting scans for project '{project_code}'") + + payload = { + "group": "projects", + "action": "get_all_scans", + "data": { + "project_code": project_code + } + } + + response = self._send_request(payload) + if response.get("status") == "1" and "data" in response: + scans = response["data"] + if isinstance(scans, list): + logger.debug(f"Found {len(scans)} scans for project '{project_code}'") + return scans + elif isinstance(scans, dict): + # Sometimes API returns dict instead of list + logger.debug(f"Found {len(scans)} scans for project '{project_code}' (as dict)") + return list(scans.values()) if scans else [] + else: + logger.warning(f"Expected list or dict for project scans, got {type(scans)}") + return [] + else: + error_msg = response.get("error", f"Unexpected response: {response}") + # Don't raise error for project not found - just return empty list + if "Project does not exist" in error_msg or "row_not_found" in error_msg: + logger.debug(f"Project '{project_code}' not found, returning empty scan list") + return [] + raise ApiError(f"Failed to get scans for project '{project_code}': {error_msg}", details=response) + + def create_project(self, project_code: str): + """ + Create new project. + Enhanced with better validation and error handling. + + Args: + project_code: The unique identifier for the project + + Raises: + ProjectExistsError: If a project with this code already exists + ValidationError: If parameters are invalid + ApiError: If the API call fails + NetworkError: If there are network issues + """ + # Enhanced validation + self._validate_project_parameters(project_code) + + logger.debug(f"Creating project '{project_code}'") + + payload = { + "group": "projects", + "action": "create", + "data": { + "project_code": project_code, + "project_name": project_code, + "description": "Automatically created by Workbench Agent script", + }, + } + + try: + response = self._send_request(payload) + if response.get("status") == "1": + logger.info(f"Successfully created project '{project_code}'") + print(f"Created project {project_code}") # Match original behavior for tests + else: + error_msg = response.get("error", f"Unexpected response: {response}") + raise ApiError( + f"Failed to create project '{project_code}': {error_msg}", details=response + ) + except ProjectExistsError: + raise # Re-raise specific errors + except Exception as e: + if isinstance(e, (ApiError, ProjectExistsError)): + raise + raise ApiError( + f"Failed to create project '{project_code}': {e}", details={"error": str(e)} + ) + + def projects_get_policy_warnings_info(self, project_code: str) -> Dict[str, Any]: + """ + Retrieve policy warnings information at project level. + + Args: + project_code: The unique identifier for the project + + Returns: + dict: The policy warnings data + + Raises: + ProjectNotFoundError: If the project doesn't exist + ApiError: If there are API issues + NetworkError: If there are network issues + """ + logger.debug(f"Getting policy warnings info for project '{project_code}'") + + payload = { + "group": "projects", + "action": "get_policy_warnings_info", + "data": { + "project_code": project_code, + }, + } + + response = self._send_request(payload) + if response.get("status") == "1" and "data" in response: + return response["data"] + else: + error_msg = response.get("error", "Unknown error") + if "Project does not exist" in error_msg or "row_not_found" in error_msg: + raise ProjectNotFoundError(f"Project '{project_code}' not found") + raise ApiError( + f"Failed to get policy warnings info for project '{project_code}': {error_msg}", + details=response, + ) diff --git a/src/workbench_agent/api/scans_api.py b/src/workbench_agent/api/scans_api.py new file mode 100644 index 0000000..56200dd --- /dev/null +++ b/src/workbench_agent/api/scans_api.py @@ -0,0 +1,770 @@ +import logging +from typing import Dict, Any, List, Optional, Tuple +from .helpers.api_base import APIBase +from .helpers.process_waiters import ProcessWaiters +from .helpers.status_checkers import StatusCheckers +from ..exceptions import ApiError, ScanNotFoundError, ScanExistsError, ValidationError + +logger = logging.getLogger("workbench-agent") + + +class ScansAPI(APIBase, ProcessWaiters, StatusCheckers): + """ + API client for scan-related operations on the Workbench platform. + + This class provides methods for creating, managing, and monitoring scans, + including uploading files, running analyses, and retrieving results. + Inherits from ProcessWaiters and StatusCheckers mixins for status checking and waiting capabilities. + """ + + # --- Enhanced Validation Methods --- + + def _validate_scan_parameters(self, scan_code: str, **kwargs) -> None: + """ + Validates scan parameters before API operations. + + Args: + scan_code: The scan code to validate + **kwargs: Additional parameters to validate + + Raises: + ValidationError: If parameters are invalid + """ + if not scan_code or not scan_code.strip(): + raise ValidationError("Scan code cannot be empty") + + # Validate limit parameter + limit = kwargs.get('limit') + if limit is not None and (not isinstance(limit, int) or limit < 1): + raise ValidationError("Limit must be a positive integer") + + # Validate sensitivity parameter + sensitivity = kwargs.get('sensitivity') + if sensitivity is not None and (not isinstance(sensitivity, int) or sensitivity < 1): + raise ValidationError("Sensitivity must be a positive integer") + + # Validate match filtering threshold + threshold = kwargs.get('match_filtering_threshold') + if threshold is not None and not isinstance(threshold, int): + raise ValidationError("Match filtering threshold must be an integer") + + def _validate_reuse_parameters(self, reuse_identification: bool, identification_reuse_type: str = None, specific_code: str = None) -> None: + """ + Validates identification reuse parameters. + + Args: + reuse_identification: Whether identification reuse is enabled + identification_reuse_type: Type of reuse + specific_code: Specific code for reuse + + Raises: + ValidationError: If reuse parameters are invalid + """ + if reuse_identification: + valid_reuse_types = {"any", "only_me", "specific_project", "specific_scan"} + if identification_reuse_type and identification_reuse_type not in valid_reuse_types: + raise ValidationError(f"Invalid identification_reuse_type. Must be one of: {valid_reuse_types}") + + if identification_reuse_type in {"specific_project", "specific_scan"} and not specific_code: + raise ValidationError(f"specific_code is required when using {identification_reuse_type}") + + # --- Scan Information Methods --- + + def get_scan_information(self, scan_code: str) -> Dict[str, Any]: + """ + Retrieves detailed information about a scan. + + Args: + scan_code: Code of the scan to get information for + + Returns: + Dict containing scan information + + Raises: + ScanNotFoundError: If the scan doesn't exist + ApiError: If there are API issues + NetworkError: If there are network issues + """ + logger.debug(f"Getting scan information for '{scan_code}'") + + payload = { + "group": "scans", + "action": "get_information", + "data": { + "scan_code": scan_code, + }, + } + + response = self._send_request(payload) + if response.get("status") == "1" and "data" in response: + return response["data"] + else: + error_msg = response.get("error", "Unknown error") + if "row_not_found" in error_msg or "Scan not found" in error_msg: + raise ScanNotFoundError(f"Scan '{scan_code}' not found") + raise ApiError( + f"Failed to get scan information for '{scan_code}': {error_msg}", + details=response, + ) + + def check_if_scan_exists(self, scan_code: str) -> bool: + """ + Check if scan exists (backwards compatibility with original agent). + + Args: + scan_code: The unique identifier for the scan + + Returns: + bool: True if scan exists, False otherwise + """ + try: + self.get_scan_information(scan_code) + return True + except ScanNotFoundError: + return False + except (ApiError, Exception): + # On other errors, assume scan doesn't exist for safety + return False + + # --- Existing Methods with Enhanced Organization --- + + def list_scans(self) -> List[Dict[str, Any]]: + """ + List all scans accessible to the current user. + + Returns: + List[Dict]: List of scan dictionaries with keys like code, name, project_code, etc. + + Raises: + ApiError: If the API call fails + NetworkError: If there are network issues + """ + logger.debug("Listing all scans") + + payload = { + "group": "scans", + "action": "get_all_scans", + "data": {} + } + + response = self._send_request(payload) + if response.get("status") == "1" and "data" in response: + scans = response["data"] + if isinstance(scans, list): + logger.debug(f"Found {len(scans)} scans") + return scans + elif isinstance(scans, dict): + # Sometimes API returns dict instead of list + logger.debug(f"Found {len(scans)} scans (as dict)") + return list(scans.values()) if scans else [] + else: + logger.warning(f"Expected list or dict for scans, got {type(scans)}") + return [] + else: + error_msg = response.get("error", f"Unexpected response: {response}") + raise ApiError(f"Failed to list scans: {error_msg}", details=response) + + def create_webapp_scan( + self, scan_code: str, project_code: str = None, target_path: str = None + ) -> int: + """ + Creates a Scan in Workbench. The scan can optionally be created inside a Project. + Enhanced with better validation and error handling. + + Args: + scan_code: The unique identifier for the scan + project_code: The project code within which to create the scan + target_path: Optional target path where scan is stored (for server-side scanning) + + Returns: + int: The scan ID of the created scan + + Raises: + ScanExistsError: If a scan with this code already exists + ValidationError: If parameters are invalid + ApiError: If the API call fails + NetworkError: If there are network issues + """ + # Enhanced validation + if not scan_code or not scan_code.strip(): + raise ValidationError("Scan code cannot be empty") + + logger.debug(f"Creating webapp scan '{scan_code}' in project '{project_code}'") + + payload = { + "group": "scans", + "action": "create", + "data": { + "scan_code": scan_code, + "scan_name": scan_code, + "project_code": project_code, + "description": "Scan created using the Workbench Agent.", + }, + } + + # Add target_path only if provided (backwards compatibility) + if target_path: + payload["data"]["target_path"] = target_path + + try: + response = self._send_request(payload) + if response.get("status") == "1" and "data" in response: + scan_id = response["data"].get("scan_id") + if scan_id is None: + raise ApiError("Scan created but no scan_id returned", details=response) + logger.debug(f"Successfully created scan '{scan_code}' with ID {scan_id}") + return int(scan_id) + else: + error_msg = response.get("error", f"Unexpected response: {response}") + raise ApiError( + f"Failed to create scan '{scan_code}': {error_msg}", details=response + ) + except ScanExistsError: + raise # Re-raise specific errors + except Exception as e: + if isinstance(e, (ApiError, ScanExistsError)): + raise + raise ApiError(f"Failed to create scan '{scan_code}': {e}", details={"error": str(e)}) + + def start_dependency_analysis(self, scan_code: str): + """ + Initiate dependency analysis for a scan. + + Args: + scan_code: The unique identifier for the scan + + Raises: + ScanNotFoundError: If the scan doesn't exist + ProcessError: If dependency analysis cannot be started + ApiError: If the API call fails + NetworkError: If there are network issues + """ + logger.debug(f"Starting dependency analysis for scan '{scan_code}'") + + payload = { + "group": "scans", + "action": "run_dependency_analysis", + "data": { + "scan_code": scan_code, + }, + } + + response = self._send_request(payload) + if response.get("status") != "1": + error_msg = response.get("error", "Unknown error") + if "Scan not found" in error_msg or "row_not_found" in error_msg: + raise ScanNotFoundError(f"Scan '{scan_code}' not found") + raise ApiError( + f"Failed to start dependency analysis for scan '{scan_code}': {error_msg}", + details=response, + ) + + logger.info(f"Dependency analysis started for scan '{scan_code}'") + + def get_pending_files(self, scan_code: str) -> Dict[str, str]: + """ + Call API scans -> get_pending_files. + + Args: + scan_code: The unique identifier for the scan + + Returns: + dict: Dictionary of pending files + + Raises: + ApiError: If there are API issues + NetworkError: If there are network issues + """ + logger.debug(f"Getting pending files for scan '{scan_code}'") + + payload = { + "group": "scans", + "action": "get_pending_files", + "data": { + "scan_code": scan_code, + }, + } + + response = self._send_request(payload) + if response.get("status") == "1" and "data" in response: + data = response["data"] + if isinstance(data, dict): + logger.debug(f"Found {len(data)} pending files for scan '{scan_code}'") + return data + else: + logger.warning(f"Expected dict for pending files, got {type(data)}") + return {} + else: + error_msg = response.get("error", f"Unexpected response: {response}") + logger.error(f"Failed to get pending files for scan '{scan_code}': {error_msg}") + return {} # Return empty dict instead of raising exception + + def get_policy_warnings_counter(self, scan_code: str) -> Dict[str, Any]: + """ + Retrieve policy warnings information at scan level. + + Args: + scan_code: The unique identifier for the scan + + Returns: + dict: The policy warnings data + + Raises: + ScanNotFoundError: If the scan doesn't exist + ApiError: If there are API issues + NetworkError: If there are network issues + """ + logger.debug(f"Getting policy warnings counter for scan '{scan_code}'") + + payload = { + "group": "scans", + "action": "get_policy_warnings_counter", + "data": { + "scan_code": scan_code, + }, + } + + response = self._send_request(payload) + if response.get("status") == "1" and "data" in response: + return response["data"] + else: + error_msg = response.get("error", "Unknown error") + if "Scan not found" in error_msg or "row_not_found" in error_msg: + raise ScanNotFoundError(f"Scan '{scan_code}' not found") + raise ApiError( + f"Failed to get policy warnings counter for scan '{scan_code}': {error_msg}", + details=response, + ) + + def get_scan_identified_components(self, scan_code: str) -> Dict[str, Any]: + """ + Retrieve the list of identified components from one scan. + + Args: + scan_code: The unique identifier for the scan + + Returns: + dict: The identified components data + + Raises: + ScanNotFoundError: If the scan doesn't exist + ApiError: If there are API issues + NetworkError: If there are network issues + """ + logger.debug(f"Getting identified components for scan '{scan_code}'") + + payload = { + "group": "scans", + "action": "get_scan_identified_components", + "data": { + "scan_code": scan_code, + }, + } + + response = self._send_request(payload) + if response.get("status") == "1" and "data" in response: + return response["data"] + else: + error_msg = response.get("error", "Unknown error") + if "Scan not found" in error_msg or "row_not_found" in error_msg: + raise ScanNotFoundError(f"Scan '{scan_code}' not found") + raise ApiError( + f"Failed to get identified components for scan '{scan_code}': {error_msg}", + details=response, + ) + + def get_scan_identified_licenses(self, scan_code: str) -> Dict[str, Any]: + """ + Retrieve the list of identified licenses from one scan. + + Args: + scan_code: The unique identifier for the scan + + Returns: + dict: The identified licenses data + + Raises: + ScanNotFoundError: If the scan doesn't exist + ApiError: If there are API issues + NetworkError: If there are network issues + """ + logger.debug(f"Getting identified licenses for scan '{scan_code}'") + + payload = { + "group": "scans", + "action": "get_scan_identified_licenses", + "data": { + "scan_code": scan_code, + "unique": "1", + }, + } + + response = self._send_request(payload) + if response.get("status") == "1" and "data" in response: + return response["data"] + else: + error_msg = response.get("error", "Unknown error") + if "Scan not found" in error_msg or "row_not_found" in error_msg: + raise ScanNotFoundError(f"Scan '{scan_code}' not found") + raise ApiError( + f"Failed to get identified licenses for scan '{scan_code}': {error_msg}", + details=response, + ) + + def get_results(self, scan_code: str) -> Dict[str, Any]: + """ + Retrieve the list matches from one scan. + + Args: + scan_code: The unique identifier for the scan + + Returns: + dict: The scan results data + + Raises: + ScanNotFoundError: If the scan doesn't exist + ApiError: If there are API issues + NetworkError: If there are network issues + """ + logger.debug(f"Getting scan results for scan '{scan_code}'") + + payload = { + "group": "scans", + "action": "get_results", + "data": { + "scan_code": scan_code, + "unique": "1", + }, + } + + response = self._send_request(payload) + if response.get("status") == "1" and "data" in response: + return response["data"] + else: + error_msg = response.get("error", "Unknown error") + if "Scan not found" in error_msg or "row_not_found" in error_msg: + raise ScanNotFoundError(f"Scan '{scan_code}' not found") + raise ApiError( + f"Failed to get scan results for scan '{scan_code}': {error_msg}", details=response + ) + + def get_dependency_analysis_result(self, scan_code: str) -> Dict[str, Any]: + """ + Retrieve dependency analysis results. + + Args: + scan_code: The unique identifier for the scan + + Returns: + dict: The dependency analysis results + + Raises: + ScanNotFoundError: If the scan doesn't exist + ApiError: If there are API issues + NetworkError: If there are network issues + """ + logger.debug(f"Getting dependency analysis results for scan '{scan_code}'") + + payload = { + "group": "scans", + "action": "get_dependency_analysis_results", + "data": { + "scan_code": scan_code, + }, + } + + response = self._send_request(payload) + if response.get("status") == "1" and "data" in response: + return response["data"] + else: + error_msg = response.get("error", "Unknown error") + if "Scan not found" in error_msg or "row_not_found" in error_msg: + raise ScanNotFoundError(f"Scan '{scan_code}' not found") + raise ApiError( + f"Failed to get dependency analysis results for scan '{scan_code}': {error_msg}", + details=response, + ) + + def extract_archives( + self, + scan_code: str, + recursively_extract_archives: bool, + jar_file_extraction: bool, + ) -> bool: + """ + Extract archive + + Args: + scan_code: The unique identifier for the scan + recursively_extract_archives: Yes or no + jar_file_extraction: Yes or no + + Returns: + bool: True for successful API call + + Raises: + ScanNotFoundError: If the scan doesn't exist + ApiError: If the API call fails + NetworkError: If there are network issues + """ + logger.debug(f"Extracting archives for scan '{scan_code}'") + + payload = { + "group": "scans", + "action": "extract_archives", + "data": { + "scan_code": scan_code, + "recursively_extract_archives": recursively_extract_archives, + "jar_file_extraction": jar_file_extraction, + }, + } + + response = self._send_request(payload) + if response.get("status") != "1": + error_msg = response.get("error", "Unknown error") + if "Scan not found" in error_msg or "row_not_found" in error_msg: + raise ScanNotFoundError(f"Scan '{scan_code}' not found") + raise ApiError( + f"Failed to extract archives for scan '{scan_code}': {error_msg}", details=response + ) + + logger.info(f"Archive extraction completed for scan '{scan_code}'") + return True + + + + def run_scan( + self, + scan_code: str, + limit: int, + sensitivity: int, + auto_identification_detect_declaration: bool, + auto_identification_detect_copyright: bool, + auto_identification_resolve_pending_ids: bool, + delta_only: bool, + reuse_identification: bool, + identification_reuse_type: str = None, + specific_code: str = None, + advanced_match_scoring: bool = True, + match_filtering_threshold: int = -1, + ): + """ + Run a scan with the specified parameters. + Enhanced with parameter validation and better error handling. + + Args: + scan_code: Unique scan identifier + limit: Limit the number of matches against the KB + sensitivity: Result sensitivity + auto_identification_detect_declaration: Automatically detect license declaration inside files + auto_identification_detect_copyright: Automatically detect copyright statements inside files + auto_identification_resolve_pending_ids: Automatically resolve pending identifications + delta_only: Scan only new or modified files + reuse_identification: Reuse previous identifications + identification_reuse_type: Possible values: any,only_me,specific_project,specific_scan + specific_code: Fill only when reuse type: specific_project or specific_scan + advanced_match_scoring: If true, scan will run with advanced match scoring + match_filtering_threshold: Minimum length (in characters) of snippet to be considered valid + + Raises: + ScanNotFoundError: If the scan doesn't exist + ValidationError: If parameters are invalid + ProcessError: If the scan cannot be started + ApiError: If the API call fails + NetworkError: If there are network issues + """ + # Enhanced parameter validation + self._validate_scan_parameters( + scan_code=scan_code, + limit=limit, + sensitivity=sensitivity, + match_filtering_threshold=match_filtering_threshold + ) + self._validate_reuse_parameters( + reuse_identification=reuse_identification, + identification_reuse_type=identification_reuse_type, + specific_code=specific_code + ) + + logger.info(f"Starting scan '{scan_code}'") + + payload = { + "group": "scans", + "action": "run", + "data": { + "scan_code": scan_code, + "limit": limit, + "sensitivity": sensitivity, + "auto_identification_detect_declaration": int( + auto_identification_detect_declaration + ), + "auto_identification_detect_copyright": int(auto_identification_detect_copyright), + "auto_identification_resolve_pending_ids": int( + auto_identification_resolve_pending_ids + ), + "delta_only": int(delta_only), + "advanced_match_scoring": int(advanced_match_scoring), + }, + } + + if match_filtering_threshold > -1: + payload["data"]["match_filtering_threshold"] = match_filtering_threshold + + if reuse_identification: + data = payload["data"] + data["reuse_identification"] = "1" + # 'any', 'only_me', 'specific_project', 'specific_scan' + if identification_reuse_type in {"specific_project", "specific_scan"}: + data["identification_reuse_type"] = identification_reuse_type + data["specific_code"] = specific_code + else: + data["identification_reuse_type"] = identification_reuse_type + + response = self._send_request(payload) + if response.get("status") != "1": + error_msg = response.get("error", "Unknown error") + logger.error(f"Failed to start scan '{scan_code}': {error_msg} payload {payload}") + raise ApiError(f"Failed to start scan '{scan_code}': {error_msg}", details=response) + + logger.info(f"Scan '{scan_code}' started successfully") + return response + + # --- Backwards Compatibility Methods --- + + def _get_scan_status(self, scan_type: str, scan_code: str) -> Dict[str, Any]: + """ + Calls API scans -> check_status (backwards compatibility with original agent). + + Args: + scan_type: One of these: SCAN, DEPENDENCY_ANALYSIS + scan_code: The unique identifier for the scan + + Returns: + dict: The data section from the JSON response returned from API + + Raises: + ApiError: If the API call fails + ScanNotFoundError: If the scan doesn't exist + """ + return self.check_status(scan_type, scan_code) + + + + def _get_pending_files(self, scan_code: str) -> Dict[str, str]: + """ + Call API scans -> get_pending_files (backwards compatibility with original agent). + + Args: + scan_code: The unique identifier for the scan + + Returns: + dict: Dictionary of pending files + + Raises: + Exception: If there are API issues (original agent behavior) + """ + try: + return self.get_pending_files(scan_code) + except Exception as e: + # Match original agent behavior - raise Exception instead of specific errors + raise Exception(f"Error getting pending files result: {e}") + + def _get_dependency_analysis_result(self, scan_code: str) -> Dict[str, Any]: + """ + Retrieve dependency analysis results (backwards compatibility with original agent). + + Args: + scan_code: The unique identifier for the scan + + Returns: + dict: The dependency analysis results + + Raises: + Exception: If there are API issues (original agent behavior) + """ + try: + return self.get_dependency_analysis_result(scan_code) + except Exception as e: + # Match original agent behavior - raise Exception instead of specific errors + raise Exception(f"Error getting dependency analysis result: {e}") + + def _cancel_scan(self, scan_code: str) -> None: + """ + Cancel a scan (backwards compatibility with original agent). + + Args: + scan_code: The unique identifier for the scan + + Raises: + Exception: If cancellation fails (original agent behavior) + """ + logger.debug(f"Cancelling scan '{scan_code}'") + + payload = { + "group": "scans", + "action": "cancel_run", + "data": { + "scan_code": scan_code, + }, + } + + response = self._send_request(payload) + if response.get("status") != "1": + # Match original agent behavior - raise Exception with specific message + raise Exception(f"Error cancelling scan: {response}") + + logger.info(f"Successfully cancelled scan '{scan_code}'") + + def scans_get_policy_warnings_counter(self, scan_code: str) -> Dict[str, Any]: + """ + Retrieve policy warnings information at scan level (backwards compatibility). + + Args: + scan_code: The unique identifier for the scan + + Returns: + dict: The policy warnings data + + Raises: + Exception: If there are API issues (original agent behavior) + """ + try: + return self.get_policy_warnings_counter(scan_code) + except Exception as e: + # Match original agent behavior - raise Exception instead of specific errors + raise Exception(f"Error getting project policy warnings information result: {e}") + + + + def remove_uploaded_content(self, filename: str, scan_code: str): + """ + When using chunked uploading every new chunk is appended to existing file, for this reason we need to make sure + that initially there is no file (from previous uploading). + + Args: + filename: The file to be deleted + scan_code: The unique identifier for the scan + """ + logger.debug(f"Removing uploaded content '{filename}' from scan '{scan_code}'") + print( + f"Called scans->remove_uploaded_content on file {filename}" + ) # Match original behavior + + payload = { + "group": "scans", + "action": "remove_uploaded_content", + "data": { + "scan_code": scan_code, + "filename": filename, + }, + } + + response = self._send_request(payload) + if response.get("status") != "1": + warning_msg = f"Cannot delete file {filename}, maybe is the first time when uploading this file? API response {response}." + print(warning_msg) # Match original behavior + logger.warning( + f"Cannot delete file '{filename}' from scan '{scan_code}', maybe is the first time uploading? API response: {response}" + ) + else: + logger.debug(f"Successfully removed '{filename}' from scan '{scan_code}'") diff --git a/src/workbench_agent/api/upload_api.py b/src/workbench_agent/api/upload_api.py new file mode 100644 index 0000000..0a369f8 --- /dev/null +++ b/src/workbench_agent/api/upload_api.py @@ -0,0 +1,61 @@ +import base64 +import os +import logging +from .helpers.upload_helpers import UploadHelper +from ..exceptions import FileSystemError + +logger = logging.getLogger("workbench-agent") + + +class UploadAPI(UploadHelper): + """ + Workbench API Upload Operations - handles file uploads with enhanced reliability. + """ + + def upload_files(self, scan_code: str, path: str, chunked_upload: bool = False): + """ + Uploads files to the Workbench using the API's File Upload endpoint with enhanced reliability. + + Args: + scan_code: The scan code where the file or files will be uploaded + path: Path to the file or files to upload + chunked_upload: Enable/disable chunk upload + + Raises: + FileSystemError: If there are file system errors + NetworkError: If there are network issues + ApiError: If the upload fails + ScanNotFoundError: If the scan doesn't exist + """ + logger.info(f"Uploading file '{path}' to scan '{scan_code}'") + + if not os.path.exists(path): + raise FileSystemError(f"File '{path}' does not exist") + + file_size = os.path.getsize(path) + filename = os.path.basename(path) + + # Prepare parameters + filename_base64 = base64.b64encode(filename.encode()).decode("utf-8") + scan_code_base64 = base64.b64encode(scan_code.encode()).decode("utf-8") + + # Check if we should use chunked upload + use_chunked = chunked_upload and (file_size > self.CHUNKED_UPLOAD_THRESHOLD) + + if use_chunked: + # First delete possible existing files because chunk uploading works by appending existing file on disk + if hasattr(self, "remove_uploaded_content"): + self.remove_uploaded_content(filename, scan_code) + else: + logger.debug( + f"remove_uploaded_content not available - chunked upload may append to existing file" + ) + + # Prepare headers + headers = { + "FOSSID-SCAN-CODE": scan_code_base64, + "FOSSID-FILE-NAME": filename_base64, + } + + # Use the helper's unified upload method + self._perform_upload(path, headers) diff --git a/src/workbench_agent/api/workbench_api.py b/src/workbench_agent/api/workbench_api.py new file mode 100644 index 0000000..a40d595 --- /dev/null +++ b/src/workbench_agent/api/workbench_api.py @@ -0,0 +1,96 @@ +import logging +import argparse +from typing import Tuple, Optional +from .projects_api import ProjectsAPI +from .scans_api import ScansAPI +from .upload_api import UploadAPI +from .helpers.project_scan_resolvers import ResolveWorkbenchProjectScan + +logger = logging.getLogger("workbench-agent") + + +class WorkbenchAPI(ProjectsAPI, ScansAPI, UploadAPI): + """ + A comprehensive client for interacting with the FossID Workbench API. + + This class composes all individual API components into a single unified client, + providing access to all Workbench functionality including: + - Project and Scan management + - Scan operations + - File uploads + + The client follows modern Python practices with: + - Comprehensive error handling with specific exception types + - Structured logging throughout all operations + - Type hints for better code clarity + - Robust network error handling and retry logic + + Attributes: + api_url: The base URL of the Workbench API + api_user: The username used for API authentication + api_token: The API token for authentication + """ + + def __init__(self, api_url: str, api_user: str, api_token: str): + """ + Initializes the Workbench API client with authentication credentials. + + Args: + api_url: The base URL of the Workbench API (will be adjusted to end with api.php if needed) + api_user: The username used for API authentication + api_token: The API token for authentication + + Note: + The API URL will be automatically adjusted to end with '/api.php' if it doesn't already. + A warning will be logged if this adjustment is made. + """ + super().__init__(api_url, api_user, api_token) + logger.info(f"Initialized Workbench API client for {self.api_url}") + logger.debug(f"API user: {api_user}") + + # Initialize resolver for name-based resolution + self._resolver = ResolveWorkbenchProjectScan(api_url, api_user, api_token) + + def resolve_project(self, project_name: str, create_if_missing: bool = False) -> str: + """ + Resolve a project name to a project code, optionally creating it if not found. + + Args: + project_name: The project name to resolve + create_if_missing: Whether to create the project if it doesn't exist + + Returns: + str: The project code + """ + return self._resolver.resolve_project(project_name, create_if_missing) + + def resolve_scan(self, scan_name: str, project_name: Optional[str], create_if_missing: bool, params: argparse.Namespace, import_from_report: bool = False) -> Tuple[str, int]: + """ + Resolve a scan name to a scan code and ID, optionally creating it if not found. + + Args: + scan_name: The scan name to resolve + project_name: The project name (optional for global search) + create_if_missing: Whether to create the scan if it doesn't exist + params: Command line parameters + import_from_report: Whether this is for importing from a report + + Returns: + Tuple[str, int]: The scan code and scan ID + """ + return self._resolver.resolve_scan(scan_name, project_name, create_if_missing, params, import_from_report) + + def prepare_project_and_scan(self, project_identifier: str, scan_identifier: str, params: argparse.Namespace = None) -> Tuple[str, int]: + """ + Ensures project exists and creates scan if needed. + Supports both legacy code-based approach and new name-based approach with automatic resolution. + + Args: + project_identifier: Project code/name to use + scan_identifier: Scan code/name to use + params: Optional command line parameters for name resolution + + Returns: + Tuple[str, int]: The project code and scan ID + """ + return self._resolver.prepare_project_and_scan(project_identifier, scan_identifier, params) diff --git a/src/workbench_agent/cli.py b/src/workbench_agent/cli.py new file mode 100644 index 0000000..5ab964d --- /dev/null +++ b/src/workbench_agent/cli.py @@ -0,0 +1,428 @@ +# workbench_agent/cli.py + +import argparse +import os +import sys +import logging +import warnings +from argparse import RawTextHelpFormatter + +from .exceptions import ValidationError + +logger = logging.getLogger(__name__) + + +def create_base_parser(): + """ + Create the base parser with common arguments. + + Returns: + argparse.ArgumentParser: Base parser with common arguments + """ + # Define a custom type function which will verify for empty string + def non_empty_string(s): + if not s.strip(): + raise argparse.ArgumentTypeError("Argument cannot be empty or just whitespace.") + return s + + parser = argparse.ArgumentParser( + description="FossID Workbench Agent - Modular API client for automated scanning", + formatter_class=RawTextHelpFormatter, + epilog=""" +Environment Variables for Credentials: + WORKBENCH_URL : API Endpoint URL (e.g., https://workbench.example.com/api.php) + WORKBENCH_USER : Workbench Username + WORKBENCH_TOKEN : Workbench API Token + +Example Usage (Recommended - using names): + # Standard scan + workbench-agent scan --project-name "My Project" --scan-name "v1.0.0-scan" --path ./src --run_dependency_analysis + + # Blind scan + workbench-agent blind-scan --project-name "My Project" --scan-name "v1.0.0-blind" --path ./src + + # Dependency analysis only + workbench-agent scan --project-name "My Project" --scan-name "v1.0.0-deps" --run_only_dependency_analysis + +Example Usage (Legacy - using codes): + # Standard scan (deprecated) + workbench-agent scan --project_code MYPROJ --scan_code MYSCAN01 --path ./src --run_dependency_analysis + + # Blind scan (deprecated) + workbench-agent blind-scan --project_code MYPROJ --scan_code MYSCAN01 --path ./src + +Example Usage (Legacy Style - maintains backwards compatibility): + # Standard scan (same as before) + workbench-agent --project_code MYPROJ --scan_code MYSCAN01 --path ./src --run_dependency_analysis + + # Blind scan (same as before) + workbench-agent --project_code MYPROJ --scan_code MYSCAN01 --path ./src --blind_scan +""" + ) + + # Required arguments + required = parser.add_argument_group("Required Arguments") + required.add_argument( + "--api_url", + help="API Endpoint URL (e.g., https://workbench.example.com/api.php). Overrides WORKBENCH_URL env var.", + default=os.getenv("WORKBENCH_URL"), + required=not os.getenv("WORKBENCH_URL"), + type=non_empty_string, + metavar="URL" + ) + required.add_argument( + "--api_user", + help="Workbench Username. Overrides WORKBENCH_USER env var.", + default=os.getenv("WORKBENCH_USER"), + required=not os.getenv("WORKBENCH_USER"), + type=non_empty_string, + metavar="USER" + ) + required.add_argument( + "--api_token", + help="Workbench API Token. Overrides WORKBENCH_TOKEN env var.", + default=os.getenv("WORKBENCH_TOKEN"), + required=not os.getenv("WORKBENCH_TOKEN"), + type=non_empty_string, + metavar="TOKEN" + ) + + # Project identification arguments (new preferred way using names) + project_group = parser.add_argument_group("Project Identification (choose one)") + project_group.add_argument( + "--project-name", + help="Project name to associate the scan with. Projects are auto-created if they don't exist.", + type=non_empty_string, + metavar="NAME" + ) + project_group.add_argument( + "--project_code", + help="[DEPRECATED] Project code to associate the scan with. Use --project-name instead.", + type=non_empty_string, + metavar="CODE" + ) + + # Scan identification arguments (new preferred way using names) + scan_group = parser.add_argument_group("Scan Identification (choose one)") + scan_group.add_argument( + "--scan-name", + help="Scan name to create or use. Scans are auto-created if they don't exist.", + type=non_empty_string, + metavar="NAME" + ) + scan_group.add_argument( + "--scan_code", + help="[DEPRECATED] Scan code to create or use. Use --scan-name instead.", + type=non_empty_string, + metavar="CODE" + ) + + # Optional arguments + optional = parser.add_argument_group("Optional Arguments") + optional.add_argument( + "--log", + help="Logging level (Default: INFO)", + choices=["DEBUG", "INFO", "WARNING", "ERROR"], + default="WARNING", + ) + optional.add_argument( + "--path", + help="Local directory/file to upload and scan.", + type=str, + metavar="PATH" + ) + optional.add_argument( + "--limit", + help="Limits KB scan results (Default: 10)", + type=int, + default=10 + ) + optional.add_argument( + "--sensitivity", + help="Sets KB snippet sensitivity (Default: 10)", + type=int, + default=10 + ) + optional.add_argument( + "--recursively_extract_archives", + help="Recursively extract nested archives. Default false.", + action="store_true", + default=False, + ) + optional.add_argument( + "--jar_file_extraction", + help="Control default behavior related to extracting jar files. Default false.", + action="store_true", + default=False, + ) + optional.add_argument( + "--run_dependency_analysis", + help="Initiate dependency analysis after finishing scanning for matches in KB.", + action="store_true", + default=False, + ) + optional.add_argument( + "--run_only_dependency_analysis", + help="Scan only for dependencies, no results from KB.", + action="store_true", + default=False, + ) + optional.add_argument( + "--auto_identification_detect_declaration", + help="Automatically detect license declaration inside files.", + action="store_true", + default=False, + ) + optional.add_argument( + "--auto_identification_detect_copyright", + help="Automatically detect copyright statements inside files.", + action="store_true", + default=False, + ) + optional.add_argument( + "--auto_identification_resolve_pending_ids", + help="Automatically resolve pending identifications.", + action="store_true", + default=False, + ) + optional.add_argument( + "--delta_only", + help="Scan only delta (newly added files from last scan).", + action="store_true", + default=False, + ) + optional.add_argument( + "--reuse_identifications", + help="If present, try to use an existing identification depending on parameter 'identification_reuse_type'.", + action="store_true", + default=False, + ) + optional.add_argument( + "--identification_reuse_type", + help="Based on reuse type last identification found will be used for files with the same hash.", + choices=["any", "only_me", "specific_project", "specific_scan"], + default="any", + type=str, + ) + optional.add_argument( + "--specific_code", + help="The scan code used when creating the scan in Workbench.", + type=str, + ) + optional.add_argument( + "--no_advanced_match_scoring", + help="Disable advanced match scoring which by default is enabled.", + dest="advanced_match_scoring", + action="store_false", + ) + optional.add_argument( + "--match_filtering_threshold", + help="Minimum length, in characters, of the snippet to be considered valid after applying match filtering.", + type=int, + default=-1, + ) + + optional.add_argument( + "--chunked_upload", + help="For files bigger than 8 MB uploading will be done using chunks.", + action="store_true", + default=False, + ) + optional.add_argument( + "--path-result", + help="Save results to specified path", + type=str, + ) + + # CLI options for blind scan + cli_args = parser.add_argument_group("CLI Options (for blind scan)") + cli_args.add_argument( + "--cli_path", + help="Path to fossid-cli executable (Default: /usr/bin/fossid-cli)", + type=str, + default="/usr/bin/fossid-cli" + ) + cli_args.add_argument( + "--config_path", + help="Path to fossid.conf configuration file (Default: /etc/fossid.conf)", + type=str, + default="/etc/fossid.conf" + ) + + # Monitoring options + monitor_args = parser.add_argument_group("Scan Monitoring Options") + monitor_args.add_argument( + "--scan_number_of_tries", + help="Number of status checks before timeout (Default: 960)", + type=int, + default=960 + ) + monitor_args.add_argument( + "--scan_wait_time", + help="Seconds between status checks (Default: 30)", + type=int, + default=30 + ) + + return parser + + +def parse_cmdline_args(): + """ + Parse command line arguments for the Workbench Agent. + Supports both new subcommand style and legacy style for backwards compatibility. + + Returns: + argparse.Namespace: Parsed command line arguments + + Raises: + ValidationError: If required arguments are missing or invalid + """ + + # Check if we're using the old-style arguments (backwards compatibility) + # If first argument is not a known subcommand, assume legacy mode + known_subcommands = {'scan', 'blind-scan'} + use_subcommands = len(sys.argv) > 1 and sys.argv[1] in known_subcommands + + if use_subcommands: + # New subcommand style + main_parser = argparse.ArgumentParser( + description="FossID Workbench Agent - Modular API client for automated scanning", + formatter_class=RawTextHelpFormatter + ) + + # Add subcommands + subparsers = main_parser.add_subparsers(dest='command', help='Available commands') + + # Scan subcommand + scan_parser = subparsers.add_parser( + 'scan', + parents=[create_base_parser()], + add_help=False, + help='Standard scan - upload files and run KB scan' + ) + + # Blind scan subcommand + blind_scan_parser = subparsers.add_parser( + 'blind-scan', + parents=[create_base_parser()], + add_help=False, + help='Blind scan - generate hashes using CLI and upload hash file' + ) + + args = main_parser.parse_args() + + # Set the command type for handlers + if args.command == 'scan': + args.scan_type = 'scan' + elif args.command == 'blind-scan': + args.scan_type = 'blind_scan' + else: + raise ValidationError("Please specify either 'scan' or 'blind-scan' command") + + else: + # Legacy style - maintain backwards compatibility + parser = create_base_parser() + + # Add the legacy blind_scan flag + legacy_group = parser.add_argument_group("Legacy Options (backwards compatibility)") + legacy_group.add_argument( + "--blind_scan", + help="Use CLI to generate file hashes and upload hash file (legacy mode).", + action="store_true", + default=False, + ) + + args = parser.parse_args() + + # Set command and scan_type based on legacy flags + if args.blind_scan: + args.command = 'blind-scan' + args.scan_type = 'blind_scan' + else: + args.command = 'scan' + args.scan_type = 'scan' + + # Validate arguments + if not args.api_url or not args.api_user or not args.api_token: + raise ValidationError("API URL, user, and token must be provided") + + # Fix API URL if it doesn't end with '/api.php' + if args.api_url and not args.api_url.endswith('/api.php'): + if args.api_url.endswith('/'): + args.api_url = args.api_url + 'api.php' + else: + args.api_url = args.api_url + '/api.php' + + # Validate project and scan identification + project_name = getattr(args, 'project_name', None) + project_code = getattr(args, 'project_code', None) + scan_name = getattr(args, 'scan_name', None) + scan_code = getattr(args, 'scan_code', None) + + # Check that either name-based or code-based arguments are provided + if not (project_name or project_code): + raise ValidationError("Either --project-name or --project_code must be provided") + + if not (scan_name or scan_code): + raise ValidationError("Either --scan-name or --scan_code must be provided") + + # Check for conflicting arguments + if project_name and project_code: + raise ValidationError("Cannot use both --project-name and --project_code. Use --project-name (recommended)") + + if scan_name and scan_code: + raise ValidationError("Cannot use both --scan-name and --scan_code. Use --scan-name (recommended)") + + # Track which style was used for proper resolution logic + args.use_name_resolution = bool(project_name or scan_name) + + # Add deprecation warnings for old arguments + if project_code: + warnings.warn( + "--project_code is deprecated and will be removed in a future version. " + "Please use --project-name instead.", + DeprecationWarning, + stacklevel=2 + ) + + if scan_code: + warnings.warn( + "--scan_code is deprecated and will be removed in a future version. " + "Please use --scan-name instead.", + DeprecationWarning, + stacklevel=2 + ) + + # Normalize attribute names for backwards compatibility + # Only set missing attributes, don't override user's choice + if not hasattr(args, 'project_code') or args.project_code is None: + args.project_code = getattr(args, 'project_name', None) + if not hasattr(args, 'scan_code') or args.scan_code is None: + args.scan_code = getattr(args, 'scan_name', None) + if not hasattr(args, 'project_name') or args.project_name is None: + args.project_name = getattr(args, 'project_code', None) + if not hasattr(args, 'scan_name') or args.scan_name is None: + args.scan_name = getattr(args, 'scan_code', None) + + # Validate that path is provided unless it's dependency analysis only + if (not args.run_only_dependency_analysis and + not args.path): + raise ValidationError("Path is required unless using --run_only_dependency_analysis") + + # Validate path exists if provided + if args.path and not os.path.exists(args.path): + raise ValidationError(f"Path does not exist: {args.path}") + + # Validate mutually exclusive options + if args.run_dependency_analysis and args.run_only_dependency_analysis: + raise ValidationError("Cannot use both --run_dependency_analysis and --run_only_dependency_analysis") + + # Ensure path_result attribute exists for result handler compatibility + if hasattr(args, 'path_result'): + # Keep the existing name for compatibility + pass + else: + args.path_result = None + + return args \ No newline at end of file diff --git a/src/workbench_agent/exceptions.py b/src/workbench_agent/exceptions.py new file mode 100644 index 0000000..b980edf --- /dev/null +++ b/src/workbench_agent/exceptions.py @@ -0,0 +1,224 @@ +""" +Custom exceptions for Workbench Agent operations. + +This module defines the exception hierarchy for the Workbench Agent. All exceptions +should inherit from WorkbenchAgentError to allow for easy catching of agent-specific +errors. +""" + +from typing import Optional + + +class WorkbenchAgentError(Exception): + """Base class for all Workbench Agent errors. + + All custom exceptions in this module should inherit from this class. + This allows for easy catching of any Workbench Agent-specific error. + + Attributes: + message: A human-readable error message + code: An optional error code for programmatic handling + details: Optional additional error details + """ + + def __init__(self, message: str, code: Optional[str] = None, details: Optional[dict] = None): + self.message = message + self.code = code + self.details = details or {} + super().__init__(self.message) + + +class ApiError(WorkbenchAgentError): + """Represents an error returned by the Workbench API or during API interaction. + + This is raised when the API returns an error response or when there's an + issue with the API interaction that isn't network-related. + + Example: + try: + response = api.get_scan(scan_id) + except ApiError as e: + logger.error(f"API error: {e.message} (code: {e.code})") + """ + pass + + +class NetworkError(WorkbenchAgentError): + """Represents a network-level error during API communication. + + This includes connection errors, timeouts, and other network-related issues. + + Example: + try: + response = api.upload_file(file_path) + except NetworkError as e: + logger.error(f"Network error: {e.message}") + """ + pass + + +class AuthenticationError(ApiError): + """Raised when authentication with the Workbench API fails. + + This includes invalid credentials, expired tokens, and other + authentication-related errors. + + Example: + try: + api.authenticate() + except AuthenticationError as e: + logger.error(f"Authentication failed: {e.message}") + """ + pass + + +class ValidationError(WorkbenchAgentError): + """Raised when input validation fails. + + This includes invalid file formats, unsupported options, and other + validation-related errors. + + Example: + try: + validate_input_file(file_path) + except ValidationError as e: + logger.error(f"Validation error: {e.message}") + """ + pass + + +class ConfigurationError(WorkbenchAgentError): + """Raised for invalid configuration or command-line arguments. + + This includes missing required parameters, invalid parameter values, + and configuration file errors. + + Example: + try: + validate_config(config) + except ConfigurationError as e: + logger.error(f"Configuration error: {e.message}") + """ + pass + + +class NotFoundError(ApiError): + """Base class for errors when an entity is not found via the API. + + This is raised when attempting to access a resource that doesn't exist. + """ + pass + + +class ScanNotFoundError(NotFoundError): + """Raised when a scan is not found. + + Example: + try: + scan = api.get_scan("non_existent") + except ScanNotFoundError as e: + logger.error(f"Scan not found: {e.message}") + """ + pass + + +class ProjectNotFoundError(NotFoundError): + """Raised when a project is not found. + + Example: + try: + project = api.get_project("non_existent") + except ProjectNotFoundError as e: + logger.error(f"Project not found: {e.message}") + """ + pass + + +class ResourceExistsError(ApiError): + """Base class for errors when trying to create an entity that already exists. + + This is raised when attempting to create a resource with a name that's + already in use. + """ + pass + + +class ScanExistsError(ResourceExistsError): + """Raised when trying to create a scan that already exists. + + Example: + try: + api.create_scan("existing_scan") + except ScanExistsError as e: + logger.error(f"Scan already exists: {e.message}") + """ + pass + + +class ProjectExistsError(ResourceExistsError): + """Raised when trying to create a project that already exists. + + Example: + try: + api.create_project("existing_project") + except ProjectExistsError as e: + logger.error(f"Project already exists: {e.message}") + """ + pass + + +class ProcessError(WorkbenchAgentError): + """Raised for failures during background Workbench processes. + + This includes errors during scanning, report generation, and other + long-running operations. + + Example: + try: + api.wait_for_scan_completion(scan_id) + except ProcessError as e: + logger.error(f"Process failed: {e.message}") + """ + pass + + +class ProcessTimeoutError(ProcessError): + """Raised when waiting for a process times out. + + Example: + try: + api.wait_for_scan_completion(scan_id, timeout=300) + except ProcessTimeoutError as e: + logger.error(f"Scan timed out: {e.message}") + """ + pass + + +class FileSystemError(WorkbenchAgentError): + """Raised for errors related to local file/directory operations. + + This includes file not found, permission denied, and other filesystem-related + errors. + + Example: + try: + process_directory(path) + except FileSystemError as e: + logger.error(f"File system error: {e.message}") + """ + pass + + +class CompatibilityError(WorkbenchAgentError): + """Raised when an existing scan is incompatible with the requested operation. + + This includes trying to run operations that aren't supported by the + scan's current state or configuration. + + Example: + try: + api.run_dependency_analysis(scan_id) + except CompatibilityError as e: + logger.error(f"Operation not compatible: {e.message}") + """ + pass diff --git a/src/workbench_agent/handlers/blind_scan.py b/src/workbench_agent/handlers/blind_scan.py new file mode 100644 index 0000000..59348df --- /dev/null +++ b/src/workbench_agent/handlers/blind_scan.py @@ -0,0 +1,199 @@ +import logging +import argparse +import time + +from ..api.workbench_api import WorkbenchAPI +from ..utilities.scan_workflows import ( + perform_blind_scan, + upload_scan_content, + run_kb_scan, + run_dependency_analysis, + wait_for_scan_completion, + wait_for_dependency_analysis_completion, + wait_for_scan_completion_with_duration, + wait_for_dependency_analysis_completion_with_duration, + collect_and_save_results, + collect_and_save_results_enhanced, + determine_scans_to_run, + print_operation_summary, + print_workbench_links, + format_duration, + cleanup_temp_file +) +from ..utilities.error_handling import handler_error_wrapper +from ..utilities.cli_wrapper import CliWrapper +from ..exceptions import ValidationError + +logger = logging.getLogger("workbench-agent") + + +@handler_error_wrapper +def handle_blind_scan(workbench: WorkbenchAPI, params: argparse.Namespace) -> bool: + """ + Handler for the 'blind-scan' command. Uses FossID CLI to generate file hashes, + uploads the hash file, runs KB scan, optional dependency analysis, and collects results. + + Args: + workbench: The Workbench API client instance + params: Command line parameters + + Returns: + bool: True if the operation completed successfully + + Raises: + ValidationError: If required parameters are missing or invalid + FileSystemError: If specified paths don't exist + ProcessError: If CLI execution fails + """ + print(f"\n--- Running BLIND SCAN Command ---") + + # Initialize comprehensive duration tracking + durations = { + "kb_scan": 0.0, + "dependency_analysis": 0.0, + "hash_generation": 0.0 + } + + # Validate scan parameters + if not params.path: + raise ValidationError("A path must be provided for the blind-scan command.") + + # Determine scan operations upfront + scan_operations = determine_scans_to_run(params) + logger.info(f"Scan operations to perform: {scan_operations}") + + # Resolve project and scan (find or create) - matching inspiration pattern + print("\nChecking if the Project and Scan exist or need to be created...") + if getattr(params, 'use_name_resolution', False): + # Name-based resolution + project_code = workbench.resolve_project(params.project_name, create_if_missing=True) + scan_code, scan_id = workbench.resolve_scan( + scan_name=params.scan_name, + project_name=params.project_name, + create_if_missing=True, + params=params + ) + else: + # Legacy code-based approach + scan_code = params.scan_code # For legacy, use the scan_code directly + project_code, scan_id = workbench.prepare_project_and_scan( + project_identifier=params.project_code, + scan_identifier=params.scan_code, + params=params + ) + + # Initialize CLI wrapper + cli_wrapper = CliWrapper( + cli_path=getattr(params, 'cli_path', '/usr/bin/fossid-cli'), + config_path=getattr(params, 'config_path', '/etc/fossid.conf') + ) + + # Track completion states + scan_completed = False + da_completed = False + hash_file_path = None + + try: + # Determine DA inclusion in hash generation + include_da_in_hash = scan_operations["run_dependency_analysis"] + + # Calculate timeout in minutes + timeout_minutes = (getattr(params, 'scan_number_of_tries', 960) * + getattr(params, 'scan_wait_time', 30)) // 60 + + print(f"\nGenerating file hashes using FossID CLI...") + hash_start_time = time.time() + hash_file_path = perform_blind_scan( + cli_wrapper=cli_wrapper, + path=params.path, + run_dependency_analysis=include_da_in_hash + ) + durations["hash_generation"] = time.time() - hash_start_time + print(f"Hash generation completed in {format_duration(durations['hash_generation'])}.") + + print(f"\nUploading hash file to Workbench...") + upload_scan_content( + workbench=workbench, + scan_code=params.scan_code, + path=hash_file_path, + chunked_upload=getattr(params, 'chunked_upload', False) + ) + print("Hash file uploaded successfully.") + + # Build scan options + scan_options = { + "limit": getattr(params, 'limit', 10), + "sensitivity": getattr(params, 'sensitivity', 10), + "auto_identification_detect_declaration": getattr(params, 'auto_identification_detect_declaration', False), + "auto_identification_detect_copyright": getattr(params, 'auto_identification_detect_copyright', False), + "auto_identification_resolve_pending_ids": getattr(params, 'auto_identification_resolve_pending_ids', False), + "delta_only": getattr(params, 'delta_only', False), + "reuse_identifications": getattr(params, 'reuse_identifications', False), + "identification_reuse_type": getattr(params, 'identification_reuse_type', 'any'), + "specific_code": getattr(params, 'specific_code', None), + "advanced_match_scoring": getattr(params, 'advanced_match_scoring', True), + "match_filtering_threshold": getattr(params, 'match_filtering_threshold', -1) + } + + # Handle dependency analysis only mode + if not scan_operations["run_kb_scan"] and scan_operations["run_dependency_analysis"]: + print("\nStarting Dependency Analysis only (skipping KB scan)...") + if not include_da_in_hash: + run_dependency_analysis(workbench, params.scan_code) + da_completed, durations["dependency_analysis"] = wait_for_dependency_analysis_completion_with_duration( + workbench, params.scan_code, timeout_minutes + ) + else: + print("Dependency analysis was included in hash generation - no additional DA scan needed.") + da_completed = True + + # Run KB scan if requested + elif scan_operations["run_kb_scan"]: + print("\nStarting KB Scan Process...") + run_kb_scan(workbench, params.scan_code, scan_options) + + scan_completed, durations["kb_scan"] = wait_for_scan_completion_with_duration( + workbench, params.scan_code, timeout_minutes + ) + + # Run dependency analysis if requested and not already included in hash generation + if scan_completed and scan_operations["run_dependency_analysis"] and not include_da_in_hash: + print("\nWaiting for Dependency Analysis to complete...") + run_dependency_analysis(workbench, params.scan_code) + da_completed, durations["dependency_analysis"] = wait_for_dependency_analysis_completion_with_duration( + workbench, params.scan_code, timeout_minutes + ) + elif scan_operations["run_dependency_analysis"] and include_da_in_hash: + print("Dependency analysis was included in hash generation.") + da_completed = True + + # Collect results with enhanced structure + results = collect_and_save_results_enhanced(workbench, params.scan_code, params) + + # Print comprehensive operation summary + print_operation_summary(params, scan_completed, da_completed, project_code, params.scan_code, durations) + + # Print Workbench links if available + if scan_id: + print_workbench_links(workbench.api_url, scan_id) + + # Enhanced completion summary + total_operations_time = sum(durations.values()) + result_count = results.get('metadata', {}).get('count', len(results.get('data', [])) if isinstance(results.get('data'), (list, dict)) else 0) + + print(f"\n✅ Blind scan command completed successfully!") + print(f"📊 Total operation time: {format_duration(total_operations_time)}") + print(f"📋 Found {result_count} result entries.") + + logger.info(f"Blind scan command completed successfully. Found {result_count} result entries.") + + return True + + finally: + # Cleanup temporary hash file + if hash_file_path: + cleanup_success = cleanup_temp_file(hash_file_path) + if cleanup_success: + logger.debug("Temporary hash file cleaned up successfully.") + else: + logger.warning("Failed to clean up temporary hash file.") diff --git a/src/workbench_agent/handlers/scan.py b/src/workbench_agent/handlers/scan.py new file mode 100644 index 0000000..851caeb --- /dev/null +++ b/src/workbench_agent/handlers/scan.py @@ -0,0 +1,202 @@ +import logging +import argparse +import time +from typing import Dict, Any + +from ..api.workbench_api import WorkbenchAPI +from ..utilities.scan_workflows import ( + upload_scan_content, + extract_archives, + run_kb_scan, + run_dependency_analysis, + wait_for_scan_completion, + wait_for_dependency_analysis_completion, + wait_for_scan_completion_with_duration, + wait_for_dependency_analysis_completion_with_duration, + collect_and_save_results, + collect_and_save_results_enhanced, + determine_scans_to_run, + print_operation_summary, + print_workbench_links, + format_duration +) +from ..utilities.error_handling import handler_error_wrapper +from ..exceptions import ValidationError, FileSystemError + +logger = logging.getLogger("workbench-agent") + + +@handler_error_wrapper +def handle_scan(workbench: WorkbenchAPI, params: argparse.Namespace) -> bool: + """ + Handler for the 'scan' command. Uploads code files directly, runs KB scan, + optional dependency analysis, and collects results. + + Args: + workbench: The Workbench API client instance + params: Command line parameters + + Returns: + bool: True if the operation completed successfully + + Raises: + ValidationError: If required parameters are missing or invalid + FileSystemError: If specified paths don't exist + """ + print(f"\n--- Running SCAN Command ---") + + # Initialize comprehensive duration tracking + durations = { + "kb_scan": 0.0, + "dependency_analysis": 0.0, + "extraction_duration": 0.0 + } + + # Validate scan parameters + if not params.path: + raise ValidationError("A path must be provided for the scan command.") + + # Determine scan operations upfront + scan_operations = determine_scans_to_run(params) + logger.info(f"Scan operations to perform: {scan_operations}") + + # Resolve project and scan (find or create) - matching inspiration pattern + print("\nChecking if the Project and Scan exist or need to be created...") + if getattr(params, 'use_name_resolution', False): + # Name-based resolution + project_code = workbench.resolve_project(params.project_name, create_if_missing=True) + scan_code, scan_id = workbench.resolve_scan( + scan_name=params.scan_name, + project_name=params.project_name, + create_if_missing=True, + params=params + ) + else: + # Legacy code-based approach + scan_code = params.scan_code # For legacy, use the scan_code directly + project_code, scan_id = workbench.prepare_project_and_scan( + project_identifier=params.project_code, + scan_identifier=params.scan_code, + params=params + ) + + # Enhanced upload process with clear feedback + if params.path: + # Ensure scan is idle before uploading content + print("\nEnsuring the Scan is idle before uploading code...") + workbench.ensure_scan_is_idle( + scan_code, + ["EXTRACT_ARCHIVES", "SCAN", "DEPENDENCY_ANALYSIS"], + getattr(params, 'scan_number_of_tries', 10), + getattr(params, 'scan_wait_time', 30) + ) + + print("\nClearing existing scan content...") + try: + # This method exists in the API + workbench.remove_uploaded_content("", scan_code) # Use resolved scan_code + print("Successfully cleared existing scan content.") + except Exception as e: + logger.warning(f"Failed to clear existing scan content: {e}") + print(f"Warning: Could not clear existing scan content: {e}") + print("Continuing with upload...") + + print(f"\nUploading Code to Workbench...") + upload_scan_content( + workbench=workbench, + scan_code=scan_code, # Use resolved scan_code + path=params.path, + chunked_upload=getattr(params, 'chunked_upload', False) + ) + print(f"Successfully uploaded {params.path} to Workbench.") + + print("\nExtracting Uploaded Archives...") + extract_start_time = time.time() + extract_archives( + workbench=workbench, + scan_code=scan_code, # Use resolved scan_code + recursive=getattr(params, 'recursively_extract_archives', False), + jar_extraction=getattr(params, 'jar_file_extraction', False) + ) + durations["extraction_duration"] = time.time() - extract_start_time + print("Archive extraction completed.") + + # Verify scan can start after archive extraction + print("\nVerifying scan readiness after archive extraction...") + workbench.ensure_scan_is_idle( + scan_code, + ["EXTRACT_ARCHIVES", "SCAN", "DEPENDENCY_ANALYSIS"], + getattr(params, 'scan_number_of_tries', 10), + getattr(params, 'scan_wait_time', 30) + ) + + # Track completion states + scan_completed = False + da_completed = False + + # Calculate timeout in minutes + timeout_minutes = (getattr(params, 'scan_number_of_tries', 960) * + getattr(params, 'scan_wait_time', 30)) // 60 + + # Build scan options + scan_options = { + "limit": getattr(params, 'limit', 10), + "sensitivity": getattr(params, 'sensitivity', 10), + "auto_identification_detect_declaration": getattr(params, 'auto_identification_detect_declaration', False), + "auto_identification_detect_copyright": getattr(params, 'auto_identification_detect_copyright', False), + "auto_identification_resolve_pending_ids": getattr(params, 'auto_identification_resolve_pending_ids', False), + "delta_only": getattr(params, 'delta_only', False), + "reuse_identifications": getattr(params, 'reuse_identifications', False), + "identification_reuse_type": getattr(params, 'identification_reuse_type', 'any'), + "specific_code": getattr(params, 'specific_code', None), + "advanced_match_scoring": getattr(params, 'advanced_match_scoring', True), + "match_filtering_threshold": getattr(params, 'match_filtering_threshold', -1) + } + + # Handle dependency analysis only mode + if not scan_operations["run_kb_scan"] and scan_operations["run_dependency_analysis"]: + print("\nStarting Dependency Analysis only (skipping KB scan)...") + run_dependency_analysis(workbench, scan_code) # Use resolved scan_code + + da_completed, durations["dependency_analysis"] = wait_for_dependency_analysis_completion_with_duration( + workbench, scan_code, timeout_minutes # Use resolved scan_code + ) + + # Run KB scan if requested + elif scan_operations["run_kb_scan"]: + print("\nStarting KB Scan Process...") + run_kb_scan(workbench, scan_code, scan_options) # Use resolved scan_code + + scan_completed, durations["kb_scan"] = wait_for_scan_completion_with_duration( + workbench, scan_code, timeout_minutes # Use resolved scan_code + ) + + # Run dependency analysis if requested + if scan_completed and scan_operations["run_dependency_analysis"]: + print("\nWaiting for Dependency Analysis to complete...") + run_dependency_analysis(workbench, scan_code) # Use resolved scan_code + da_completed, durations["dependency_analysis"] = wait_for_dependency_analysis_completion_with_duration( + workbench, scan_code, timeout_minutes # Use resolved scan_code + ) + + # Collect results with enhanced structure + results = collect_and_save_results_enhanced(workbench, scan_code, params) # Use resolved scan_code + + # Print comprehensive operation summary + print_operation_summary(params, scan_completed, da_completed, project_code, scan_code, durations) # Use resolved scan_code + + # Print Workbench links if available + if scan_id: + print_workbench_links(workbench.api_url, scan_id) + + # Enhanced completion summary + total_operations_time = sum(durations.values()) + result_count = results.get('metadata', {}).get('count', len(results.get('data', [])) if isinstance(results.get('data'), (list, dict)) else 0) + + print(f"\n✅ Scan command completed successfully!") + print(f"📊 Total operation time: {format_duration(total_operations_time)}") + print(f"📋 Found {result_count} result entries.") + + logger.info(f"Scan command completed successfully. Found {result_count} result entries.") + + return True diff --git a/src/workbench_agent/main.py b/src/workbench_agent/main.py new file mode 100644 index 0000000..1ed197f --- /dev/null +++ b/src/workbench_agent/main.py @@ -0,0 +1,224 @@ +import sys +import time +import logging +import traceback +from typing import Optional + +# Import from other modules in the package +from .api import WorkbenchAPI +from .cli import parse_cmdline_args +from .utilities.error_handling import agent_error_wrapper +from .utilities.scan_workflows import format_duration +from .exceptions import ( + WorkbenchAgentError, + ApiError, + NetworkError, + AuthenticationError, + ProcessError, + ProcessTimeoutError, + FileSystemError, + ValidationError, + ProjectNotFoundError, + ScanNotFoundError +) + +# Import handlers +from .handlers.scan import handle_scan +from .handlers.blind_scan import handle_blind_scan + + +def setup_logging(log_level: str) -> logging.Logger: + """ + Set up enhanced logging configuration with both file and console handlers. + + Args: + log_level: The logging level (DEBUG, INFO, WARNING, ERROR) + + Returns: + Configured logger instance + """ + # Parse log level + numeric_level = getattr(logging, log_level.upper(), logging.INFO) + + # Configure basic logging (file handler) + logging.basicConfig( + level=numeric_level, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + handlers=[logging.FileHandler("workbench-agent-log.txt", mode='w')], + force=True # Allow reconfiguration if run multiple times + ) + + # Add console handler with simpler format + console_handler = logging.StreamHandler(sys.stdout) + console_formatter = logging.Formatter('%(levelname)s: %(message)s') + console_handler.setFormatter(console_formatter) + console_handler.setLevel(numeric_level) + logging.getLogger().addHandler(console_handler) + + return logging.getLogger("workbench-agent") + + +def print_configuration(params) -> None: + """ + Print configuration summary for verification. + + Args: + params: Parsed command line parameters + """ + print("--- Workbench Agent Configuration ---") + print(f"Command: {getattr(params, 'command', getattr(params, 'scan_type', 'unknown'))}") + + # Sort and display all parameters + for k, v in sorted(params.__dict__.items()): + if k in ['command', 'scan_type']: + continue + display_val = v + + # Mask sensitive information unless in debug mode + if k == 'api_token' and getattr(params, 'log', 'INFO').upper() != 'DEBUG': + display_val = "****" if v else "Not Set" + + print(f" {k:<30} = {display_val}") + print("------------------------------------") + + +def main() -> int: + """ + Main function to parse arguments, set up logging, initialize the API client, + and execute the workbench agent operations using the appropriate handler. + + Returns: + int: Exit code (0 for success, non-zero for failure) + """ + start_time = time.monotonic() + exit_code = 1 # Default to failure + logger = None # Initialize logger variable + + try: + # Parse command line arguments + args = parse_cmdline_args() + + # Setup enhanced logging + logger = setup_logging(args.log) + + # Print configuration for verification + print_configuration(args) + + logger.info("FossID Workbench Agent starting...") + logger.debug(f"Command line arguments: {vars(args)}") + + # Initialize API client + logger.info("Initializing Workbench API client...") + api = WorkbenchAPI( + api_url=args.api_url, + api_user=args.api_user, + api_token=args.api_token + ) + logger.info("Workbench API client initialized.") + + # Command handler dispatch + COMMAND_HANDLERS = { + 'scan': handle_scan, + 'blind_scan': handle_blind_scan, + } + + # Determine which handler to use + command_key = getattr(args, 'scan_type', getattr(args, 'command', None)) + handler = COMMAND_HANDLERS.get(command_key) + + if not handler: + raise ValidationError(f"Unknown command/scan type: {command_key}") + + # Execute the command handler + logger.info(f"Executing {command_key} command...") + success = handler(api, args) + + if success: + exit_code = 0 + print("\nWorkbench Agent finished successfully.") + else: + logger.error("Handler reported failure") + print("\nWorkbench Agent finished with errors.") + exit_code = 1 + + # Enhanced exception handling with detailed error information + except (AuthenticationError, ValidationError) as e: + # Errors typically due to user input/setup + print(f"\nDetailed Error Information:") + print(f"Configuration Error: {e}") + if logger: + logger.error("%s: %s", type(e).__name__, e, exc_info=False) + exit_code = 2 + + except (ApiError, NetworkError) as e: + # API and network related errors + print(f"\nDetailed Error Information:") + print(f"API/Network Error: {e}") + if logger: + logger.error("%s: %s", type(e).__name__, e, exc_info=True) + exit_code = 3 + + except (ProcessError, ProcessTimeoutError) as e: + # Process execution errors + print(f"\nDetailed Error Information:") + print(f"Process Error: {e}") + if logger: + logger.error("%s: %s", type(e).__name__, e, exc_info=True) + exit_code = 4 + + except FileSystemError as e: + # File system related errors + print(f"\nDetailed Error Information:") + print(f"File System Error: {e}") + if logger: + logger.error("%s: %s", type(e).__name__, e, exc_info=True) + exit_code = 5 + + except (ProjectNotFoundError, ScanNotFoundError) as e: + # Resource not found errors + print(f"\nDetailed Error Information:") + print(f"Resource Error: {e}") + if logger: + logger.error("%s: %s", type(e).__name__, e, exc_info=True) + exit_code = 6 + + except WorkbenchAgentError as e: + # General workbench agent errors + print(f"\nDetailed Error Information:") + print(f"Workbench Agent Error: {e}") + if logger: + logger.error("%s: %s", type(e).__name__, e, exc_info=True) + exit_code = 7 + + except KeyboardInterrupt: + print(f"\nOperation interrupted by user") + if logger: + logger.warning("Operation interrupted by user") + exit_code = 130 + + except Exception as e: + # Catch truly unexpected errors + print(f"\nDetailed Error Information:") + print(f"Unexpected Error: {e}") + # Format and print the traceback + tb_lines = traceback.format_exception(type(e), e, e.__traceback__) + print("".join(tb_lines).rstrip()) + if logger: + logger.critical("Unexpected error occurred", exc_info=True) + exit_code = 1 + + finally: + # Calculate and print duration regardless of success/failure + end_time = time.monotonic() + duration_seconds = end_time - start_time + duration_str = format_duration(duration_seconds) + print(f"\nTotal Execution Time: {duration_str}") + if logger: + logger.info("Total execution time: %s", duration_str) + + return exit_code + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/src/workbench_agent/utilities/__init__.py b/src/workbench_agent/utilities/__init__.py new file mode 100644 index 0000000..3bc7255 --- /dev/null +++ b/src/workbench_agent/utilities/__init__.py @@ -0,0 +1,8 @@ +""" +Utility modules for the Workbench Agent. +""" + +from ..exceptions import * +from .error_handling import * +from .cli_wrapper import CliWrapper +from .result_handler import save_results \ No newline at end of file diff --git a/src/workbench_agent/utilities/cli_wrapper.py b/src/workbench_agent/utilities/cli_wrapper.py new file mode 100644 index 0000000..6d00273 --- /dev/null +++ b/src/workbench_agent/utilities/cli_wrapper.py @@ -0,0 +1,210 @@ +""" +CLI Wrapper for FossID CLI interactions. + +This module provides a wrapper for interacting with the FossID CLI tool, +particularly for blind scan functionality. +""" + +import os +import sys +import random +import logging +import subprocess +import traceback +from typing import Optional + +from ..exceptions import ProcessError, FileSystemError + +logger = logging.getLogger(__name__) + + +class CliWrapper: + """ + A class to interact with the FossID CLI. + + Attributes: + cli_path (str): Path to the executable file "fossid-cli" + config_path (str): Path to the configuration file "fossid.conf" + timeout (str): Timeout for CLI expressed in seconds + """ + + def __init__(self, cli_path: str, config_path: str, timeout: str = "120"): + """ + Initialize CliWrapper. + + Args: + cli_path: Path to the fossid-cli executable + config_path: Path to the fossid.conf configuration file + timeout: Timeout in seconds (default: "120") + + Raises: + FileSystemError: If cli_path doesn't exist or isn't executable + """ + self.cli_path = cli_path + self.config_path = config_path + self.timeout = timeout + + # Validate CLI path exists and is executable + if not os.path.exists(cli_path): + raise FileSystemError(f"FossID CLI not found at path: {cli_path}") + if not os.access(cli_path, os.X_OK): + raise FileSystemError(f"FossID CLI not executable: {cli_path}") + + logger.debug(f"CliWrapper initialized with cli_path={cli_path}, timeout={timeout}") + + def get_version(self) -> str: + """ + Get CLI version. + + Returns: + str: Version information from fossid-cli + + Raises: + ProcessError: If CLI execution fails + """ + args = [self.cli_path, "--version"] + logger.debug(f"Getting CLI version with command: {' '.join(args)}") + + try: + result = subprocess.check_output( + args, + stderr=subprocess.STDOUT, + timeout=int(self.timeout) + ) + version = result.decode('utf-8').strip() + logger.info(f"FossID CLI version: {version}") + return version + except subprocess.TimeoutExpired as e: + error_msg = f"CLI version check timed out after {self.timeout} seconds" + logger.error(error_msg) + raise ProcessError(error_msg) from e + except subprocess.CalledProcessError as e: + error_msg = f"CLI version check failed: {e.cmd} (exit code: {e.returncode})" + logger.error(error_msg) + raise ProcessError(error_msg) from e + except Exception as e: + error_msg = f"Unexpected error getting CLI version: {e}" + logger.error(error_msg) + raise ProcessError(error_msg) from e + + def blind_scan(self, path: str, run_dependency_analysis: bool = False) -> str: + """ + Call fossid-cli on a given path to generate hashes of the files from that path. + + Args: + path: Path of the code to be scanned + run_dependency_analysis: Whether to run dependency analysis or not + + Returns: + str: Path to temporary .fossid file containing generated hashes + + Raises: + FileSystemError: If the input path doesn't exist + ProcessError: If CLI execution fails + """ + if not os.path.exists(path): + raise FileSystemError(f"Scan path does not exist: {path}") + + temporary_file_path = f"/tmp/blind_scan_result_{self.randstring(8)}.fossid" + logger.info(f"Starting blind scan of path: {path}") + logger.debug(f"Temporary file will be created at: {temporary_file_path}") + + # Create temporary file, make it empty if already exists + try: + with open(temporary_file_path, "w") as f: + pass # Create empty file + except Exception as e: + raise FileSystemError(f"Failed to create temporary file {temporary_file_path}: {e}") from e + + # Build command - no longer using external timeout command + cmd_args = [self.cli_path, "--local", "--enable-sha1=1"] + + if run_dependency_analysis: + cmd_args.append("--dependency-analysis=1") + logger.debug("Dependency analysis enabled for blind scan") + + cmd_args.append(path) + logger.debug(f"Executing blind scan command: {' '.join(cmd_args)}") + + try: + # Execute command and redirect output to temporary file + with open(temporary_file_path, "w") as outfile: + result = subprocess.run( + cmd_args, + stdout=outfile, + stderr=subprocess.PIPE, + text=True, + timeout=int(self.timeout) + ) + + if result.returncode != 0: + error_msg = f"Blind scan failed with exit code {result.returncode}: {result.stderr}" + logger.error(error_msg) + # Clean up temporary file + if os.path.exists(temporary_file_path): + os.remove(temporary_file_path) + raise ProcessError(error_msg) + + # Verify temporary file was created and has content + if not os.path.exists(temporary_file_path): + raise ProcessError(f"Temporary file was not created: {temporary_file_path}") + + file_size = os.path.getsize(temporary_file_path) + if file_size == 0: + logger.warning("Blind scan completed but generated empty results file") + else: + logger.info(f"Blind scan completed successfully. Generated {file_size} bytes of hash data.") + + return temporary_file_path + + except subprocess.TimeoutExpired as e: + error_msg = f"Blind scan timed out after {self.timeout} seconds" + logger.error(error_msg) + # Clean up temporary file + if os.path.exists(temporary_file_path): + os.remove(temporary_file_path) + raise ProcessError(error_msg) from e + except Exception as e: + error_msg = f"Unexpected error during blind scan: {e}" + logger.error(error_msg) + logger.debug(traceback.format_exc()) + # Clean up temporary file + if os.path.exists(temporary_file_path): + os.remove(temporary_file_path) + raise ProcessError(error_msg) from e + + @staticmethod + def randstring(length: int = 10) -> str: + """ + Generate a random string of a given length. + + Parameters: + length: Length of the generated string (default: 10) + + Returns: + str: Random string of specified length + """ + valid_letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + return "".join((random.choice(valid_letters) for i in range(length))) + + def cleanup_temp_file(self, file_path: str) -> bool: + """ + Clean up a temporary file created by blind scan. + + Args: + file_path: Path to the temporary file to delete + + Returns: + bool: True if file was successfully deleted, False otherwise + """ + try: + if os.path.exists(file_path): + os.remove(file_path) + logger.debug(f"Cleaned up temporary file: {file_path}") + return True + else: + logger.warning(f"Temporary file does not exist: {file_path}") + return False + except Exception as e: + logger.error(f"Failed to clean up temporary file {file_path}: {e}") + return False \ No newline at end of file diff --git a/src/workbench_agent/utilities/error_handling.py b/src/workbench_agent/utilities/error_handling.py new file mode 100644 index 0000000..70b7f0b --- /dev/null +++ b/src/workbench_agent/utilities/error_handling.py @@ -0,0 +1,294 @@ +""" +Error handling utilities for the Workbench Agent. + +This module provides standardized error handling and formatting +for better user experience in CI/CD pipeline scenarios. +""" + +import logging +import argparse +import functools +from typing import Callable + +from ..exceptions import ( + WorkbenchAgentError, + ApiError, + NetworkError, + AuthenticationError, + ProcessError, + ProcessTimeoutError, + FileSystemError, + ValidationError, + ProjectNotFoundError, + ScanNotFoundError, + ConfigurationError, + CompatibilityError +) + +logger = logging.getLogger("workbench-agent") + + +def handler_error_wrapper(handler_func: Callable) -> Callable: + """ + A simple decorator for handler functions that provides logging and re-raises exceptions. + + This wrapper ensures handler functions log their execution but lets exceptions + bubble up to the main error handling logic. + + Args: + handler_func: The handler function to wrap + + Returns: + Wrapped handler function + """ + @functools.wraps(handler_func) + def wrapper(*args, **kwargs): + handler_name = handler_func.__name__ + logger.debug(f"Starting handler: {handler_name}") + + try: + result = handler_func(*args, **kwargs) + logger.debug(f"Handler {handler_name} completed successfully") + return result + except Exception as e: + logger.debug(f"Handler {handler_name} raised exception: {type(e).__name__}: {e}") + raise # Re-raise the exception to be handled by main error handling + + return wrapper + + +def format_and_print_error(error: Exception, params: argparse.Namespace): + """ + Formats and prints a standardized error message for CI/CD users. + + This centralized function handles consistent error formatting, + providing helpful guidance for common integration scenarios. + + Args: + error: The exception that occurred + params: Command line parameters + """ + error_type = type(error).__name__ + + # Get error details if available (for our custom errors) + error_message = getattr(error, 'message', str(error)) + error_code = getattr(error, 'code', None) + error_details = getattr(error, 'details', {}) + + # Add context-specific help based on error type + if isinstance(error, ProjectNotFoundError): + print(f"\n❌ Project not found") + print(f" Project '{params.project_code}' does not exist in your Workbench instance.") + print(f"\n💡 Possible solutions:") + print(f" • Check that the project name is spelled correctly") + print(f" • Verify the project exists in Workbench: {params.api_url}") + print(f" • Ensure your account has access to this project") + print(f" • The project will be created automatically if it doesn't exist") + + elif isinstance(error, ScanNotFoundError): + print(f"\n❌ Scan not found") + print(f" Scan '{params.scan_code}' does not exist in project '{params.project_code}'.") + print(f"\n💡 Possible solutions:") + print(f" • Check that the scan name is spelled correctly") + print(f" • Verify the scan exists in the specified project") + print(f" • The scan will be created automatically if it doesn't exist") + + elif isinstance(error, NetworkError): + print(f"\n❌ Network connectivity issue") + print(f" Unable to connect to the Workbench server.") + print(f" Details: {error_message}") + print(f"\n💡 Please check:") + print(f" • The Workbench server is accessible from your CI/CD environment") + print(f" • The API URL is correct: {params.api_url}") + print(f" • Network firewalls allow outbound HTTPS connections") + print(f" • The server is not experiencing downtime") + + elif isinstance(error, ApiError): + # Check for credential errors first + if "user_not_found_or_api_key_is_not_correct" in error_message: + print(f"\n❌ Invalid Workbench credentials") + print(f" The username or API token provided is incorrect.") + print(f"\n💡 Please verify:") + print(f" • Username: {params.api_user}") + print(f" • API token is correct and not expired") + print(f" • Account has access to the Workbench instance") + print(f" • API URL is correct: {params.api_url}") + print(f"\n🔧 In CI/CD pipelines:") + print(f" • Store credentials as secure environment variables") + print(f" • Ensure API tokens have sufficient permissions") + return # Exit early to avoid showing generic API error details + + print(f"\n❌ Workbench API error") + print(f" {error_message}") + + if error_code: + print(f" Error code: {error_code}") + print(f"\n💡 The Workbench API reported an issue with your request") + + elif isinstance(error, ProcessTimeoutError): + print(f"\n❌ Operation timed out") + print(f" {error_message}") + print(f"\n💡 For CI/CD environments, consider:") + print(f" • Increasing timeout values:") + print(f" --scan-number-of-tries (current: {params.scan_number_of_tries})") + print(f" --scan-wait-time (current: {params.scan_wait_time})") + print(f" • Large codebases may require longer scan times") + print(f" • Check Workbench server performance and load") + + elif isinstance(error, ProcessError): + print(f"\n❌ Workbench process failed") + print(f" {error_message}") + print(f"\n💡 Common causes:") + print(f" • Scan conflicts with existing operations") + print(f" • Server resource limitations") + print(f" • Invalid scan configuration") + + elif isinstance(error, FileSystemError): + print(f"\n❌ File system error") + print(f" {error_message}") + print(f"\n💡 Please check:") + print(f" • File and directory permissions are correct") + print(f" • Specified paths exist and are accessible") + if hasattr(params, 'path'): + print(f" • Source path: {params.path}") + if hasattr(params, 'path_result'): + print(f" • Output path: {params.path_result}") + print(f"\n🔧 In CI/CD pipelines:") + print(f" • Ensure the agent has read access to source files") + print(f" • Verify write permissions for output directories") + + elif isinstance(error, ValidationError): + print(f"\n❌ Invalid configuration") + print(f" {error_message}") + print(f"\n💡 Please check your command-line arguments:") + print(f" • All required parameters are provided") + print(f" • Parameter values are in the correct format") + print(f" • File paths are valid and accessible") + + elif isinstance(error, AuthenticationError): + print(f"\n❌ Authentication failed") + print(f" {error_message}") + print(f"\n💡 Authentication checklist:") + print(f" • API credentials are correct") + print(f" • Account has necessary permissions") + print(f" • API token is not expired") + print(f" • Account is not locked or disabled") + + elif isinstance(error, (ConfigurationError, CompatibilityError)): + # These are usually handled gracefully, but just in case + print(f"\n⚠️ Resource already exists") + print(f" {error_message}") + print(f" This is typically handled automatically - continuing with existing resource.") + + else: + # Generic error formatting for unexpected errors + print(f"\n❌ Unexpected error occurred") + print(f" {error_message}") + print(f" Error type: {error_type}") + + # Show error code if available (and not already shown) + if error_code and not isinstance(error, (ApiError,)): + print(f"\nError code: {error_code}") + + # Show details in verbose mode + if getattr(params, 'log', 'ERROR') == 'DEBUG' and error_details: + print("\n🔍 Detailed error information:") + for key, value in error_details.items(): + print(f" • {key}: {value}") + + # Add help text for debugging + if getattr(params, 'log', 'ERROR') != 'DEBUG': + print(f"\n🔧 For more detailed logs, run with --log DEBUG") + + +def agent_error_wrapper(parse_args_func: Callable) -> Callable: + """ + A decorator that wraps the main agent function with standardized error handling. + + This wrapper ensures consistent error handling for the workbench-agent, + providing user-friendly error messages while maintaining the same exit codes + and behavior expected in CI/CD environments. + + Args: + parse_args_func: Function to parse command line arguments for context + + Returns: + Decorator function + + Example: + @agent_error_wrapper(parse_cmdline_args) + def main(): + # Implementation without try/except blocks + ... + """ + def decorator(main_func: Callable) -> Callable: + @functools.wraps(main_func) + def wrapper(): + try: + logger.debug("Starting Workbench Agent execution") + + # Call the actual main function + return main_func() + + except (ProjectNotFoundError, ScanNotFoundError, FileSystemError, + ApiError, NetworkError, ProcessError, ProcessTimeoutError, + ValidationError, AuthenticationError, ConfigurationError, + CompatibilityError) as e: + # These exceptions are expected and should be handled gracefully + logger.debug(f"Expected error in workbench-agent: {type(e).__name__}: {getattr(e, 'message', str(e))}") + + # Parse command line args to get context for error formatting + try: + params = parse_args_func() + except Exception: + # Fallback if we can't parse args + params = argparse.Namespace() + params.api_url = '' + params.api_user = '' + params.project_code = '' + params.scan_code = '' + params.scan_number_of_tries = '' + params.scan_wait_time = '' + params.log = 'ERROR' + + # Format and display error message + format_and_print_error(e, params) + + # Exit with appropriate code (1 for errors, maintaining CI/CD compatibility) + logger.debug(f"Exiting with error code 1 due to {type(e).__name__}") + exit(1) + + except KeyboardInterrupt: + print(f"\n⚠️ Operation cancelled by user") + logger.debug("Operation cancelled by user (KeyboardInterrupt)") + exit(130) # Standard exit code for SIGINT + + except Exception as e: + # Unexpected errors get special handling + logger.error(f"Unexpected error in workbench-agent: {e}", exc_info=True) + + # Try to get params for context + try: + params = parse_args_func() + except Exception: + # Fallback if we can't parse args + params = argparse.Namespace() + params.log = 'ERROR' + + print(f"\n❌ Unexpected error occurred") + print(f" {str(e)}") + print(f" This may indicate a bug in the workbench-agent") + + if getattr(params, 'log', 'ERROR') == 'DEBUG': + print(f"\n🔍 Full error details:") + import traceback + traceback.print_exc() + else: + print(f"\n🔧 For full error details, run with --log DEBUG") + + # Exit with error code 2 for unexpected errors + logger.debug(f"Exiting with error code 2 due to unexpected error: {e}") + exit(2) + + return wrapper + return decorator \ No newline at end of file diff --git a/src/workbench_agent/utilities/result_handler.py b/src/workbench_agent/utilities/result_handler.py new file mode 100644 index 0000000..bfd6240 --- /dev/null +++ b/src/workbench_agent/utilities/result_handler.py @@ -0,0 +1,109 @@ +""" +Result handling utilities for the Workbench Agent. + +This module provides functionality for saving scan results to files. +""" + +import os +import json +import logging +from typing import Dict, Any +from argparse import Namespace + +logger = logging.getLogger(__name__) + + +def save_results(params: Namespace, results: Dict[str, Any]) -> None: + """ + Saves the scanning results to a specified path. + + Args: + params: Parsed command line parameters containing path_result + results: The scan results to be saved + + Note: + If params.path_result is not provided, no action is taken. + The function handles various path scenarios: + - Directory: saves as wb_results.json in the directory + - Existing file: overwrites the file + - Non-existing path: creates directories and file as needed + """ + if not hasattr(params, 'path_result') or not params.path_result: + return + + logger.debug(f"Saving results to path: {params.path_result}") + + try: + if os.path.isdir(params.path_result): + # If it's a directory, save as wb_results.json in that directory + fname = os.path.join(params.path_result, "wb_results.json") + _save_json_file(fname, results) + + elif os.path.isfile(params.path_result): + # If it's an existing file, use it directly but ensure .json extension + fname = params.path_result + _folder = os.path.dirname(params.path_result) + _fname = os.path.basename(params.path_result) + + if _fname and not _fname.endswith(".json"): + try: + # Try to replace extension with .json + if "." in _fname: + extension = _fname.split(".")[-1] + _fname = _fname.replace(f".{extension}", ".json") + else: + _fname = f"{_fname}.json" + fname = os.path.join(_folder, _fname) + except (TypeError, IndexError): + _fname = f"{_fname.replace('.', '_')}.json" + fname = os.path.join(_folder, _fname) + + os.makedirs(_folder, exist_ok=True) + _save_json_file(fname, results) + + else: + # Path doesn't exist - create it + fname = params.path_result + + if fname.endswith(".json"): + _folder = os.path.dirname(fname) + else: + if "." in fname: + _folder = os.path.dirname(fname) + else: + # Treat as directory name + _folder = fname + fname = os.path.join(_folder, "wb_results.json") + + if _folder: + os.makedirs(_folder, exist_ok=True) + _save_json_file(fname, results) + + except PermissionError as e: + logger.error(f"Permission denied when trying to save results: {e}") + print(f"Error: Permission denied when trying to save results to {params.path_result}") + except Exception as e: + logger.error(f"Error trying to save results to {params.path_result}: {e}") + print(f"Error trying to save results to {params.path_result}: {e}") + + +def _save_json_file(file_path: str, data: Dict[str, Any]) -> None: + """ + Save data to a JSON file. + + Args: + file_path: Path to save the file + data: Data to save as JSON + + Raises: + Exception: If file writing fails + """ + try: + with open(file_path, "w", encoding="utf-8") as file: + file.write(json.dumps(data, indent=4, ensure_ascii=False)) + print(f"Results saved to: {file_path}") + logger.info(f"Results successfully saved to: {file_path}") + except Exception as e: + logger.error(f"Failed to write results to {file_path}: {e}") + print(f"Error trying to write results to {file_path}") + raise \ No newline at end of file diff --git a/src/workbench_agent/utilities/scan_workflows.py b/src/workbench_agent/utilities/scan_workflows.py new file mode 100644 index 0000000..c4eedda --- /dev/null +++ b/src/workbench_agent/utilities/scan_workflows.py @@ -0,0 +1,662 @@ +import logging +import time +import os +import tempfile +from typing import Dict, Any, Tuple, Optional, Union + +from ..api.workbench_api import WorkbenchAPI +from .cli_wrapper import CliWrapper +from .result_handler import save_results +from ..exceptions import ( + WorkbenchAgentError, + ProcessError, + ProcessTimeoutError, + FileSystemError, + ValidationError +) + +logger = logging.getLogger("workbench-agent") + + + +def determine_scans_to_run(params) -> Dict[str, bool]: + """ + Determines which scan processes to run based on the provided parameters. + + Args: + params: Command line parameters + + Returns: + Dict with 'run_kb_scan' and 'run_dependency_analysis' keys + """ + run_dependency_analysis = getattr(params, 'run_dependency_analysis', False) + dependency_analysis_only = getattr(params, 'run_only_dependency_analysis', False) + + scan_operations = {"run_kb_scan": True, "run_dependency_analysis": False} + + if run_dependency_analysis and dependency_analysis_only: + logger.warning("Both --run-only-dependency-analysis and --run-dependency-analysis were specified. Using dependency-analysis-only mode (skipping KB scan).") + scan_operations["run_kb_scan"] = False + scan_operations["run_dependency_analysis"] = True + elif dependency_analysis_only: + scan_operations["run_kb_scan"] = False + scan_operations["run_dependency_analysis"] = True + elif run_dependency_analysis: + scan_operations["run_kb_scan"] = True + scan_operations["run_dependency_analysis"] = True + + logger.debug(f"Determined scan operations: {scan_operations}") + return scan_operations + + +def get_workbench_links(api_url: str, scan_id: int) -> Dict[str, Dict[str, str]]: + """ + Get all Workbench UI links and messages for a scan. + + Args: + api_url: The Workbench API URL (includes /api.php) + scan_id: The scan ID + + Returns: + Dict with link types as keys, each containing 'url' and 'message' + """ + # Link type configuration + link_config = { + "main": { + "view_param": None, + "message": "View scan results in Workbench" + }, + "pending": { + "view_param": "pending_items", + "message": "Review Pending IDs in Workbench" + }, + "policy": { + "view_param": "mark_as_identified", + "message": "Review policy warnings in Workbench" + }, + } + + # Build base URL once + base_url = api_url.replace("/api.php", "").rstrip("/") + + # Build all links + links = {} + for link_type, config in link_config.items(): + url = f"{base_url}/index.html?form=main_interface&action=scanview&sid={scan_id}" + if config["view_param"]: + url += f"¤t_view={config['view_param']}" + + links[link_type] = { + "url": url, + "message": config["message"] + } + + return links + + +def print_workbench_links(api_url: str, scan_id: int) -> None: + """ + Print convenient links to Workbench UI. + + Args: + api_url: The Workbench API URL + scan_id: The scan ID + """ + if scan_id: + print("\n--- Workbench UI Links ---") + links = get_workbench_links(api_url, scan_id) + for link_type, link_info in links.items(): + print(f"{link_info['message']}: {link_info['url']}") + print("------------------------------------") + + +def perform_blind_scan(cli_wrapper: CliWrapper, path: str, run_dependency_analysis: bool = False) -> str: + """ + Performs blind scan using CLI to generate file hashes. + + Args: + cli_wrapper: CliWrapper instance + path: Path to scan + run_dependency_analysis: Whether to include dependency analysis in hash generation + + Returns: + Path to temporary file containing generated hashes + + Raises: + ProcessError: If CLI execution fails + FileSystemError: If path doesn't exist or temp file can't be created + """ + if not os.path.exists(path): + raise FileSystemError(f"Path does not exist: {path}") + + logger.info("Performing blind scan to generate file hashes...") + + # Display CLI version for validation + try: + version = cli_wrapper.get_version() + logger.info(f"Using FossID CLI: {version}") + except Exception as e: + logger.warning(f"Could not get CLI version: {e}") + + # Generate hashes + try: + hash_file_path = cli_wrapper.blind_scan(path, run_dependency_analysis) + logger.info(f"Hash file generated at: {hash_file_path}") + return hash_file_path + except Exception as e: + raise ProcessError(f"Failed to generate hash file: {e}") + + +def upload_scan_content(workbench: WorkbenchAPI, scan_code: str, path: str, chunked_upload: bool = False) -> None: + """ + Uploads files or directories to the scan. + + Args: + workbench: WorkbenchAPI instance + scan_code: Scan code to upload to + path: Path to file or directory to upload + chunked_upload: Whether to use chunked upload for large files + + Raises: + FileSystemError: If path doesn't exist + """ + if not os.path.exists(path): + raise FileSystemError(f"Path does not exist: {path}") + + logger.info(f"Uploading content from: {path}") + + if os.path.isfile(path): + # Single file upload + logger.info(f"Uploading single file: {path}") + workbench.upload_files( + scan_code=scan_code, + path=path, + chunked_upload=chunked_upload + ) + else: + # Directory upload - upload all files + logger.info(f"Uploading directory contents: {path}") + file_count = 0 + for root, directories, filenames in os.walk(path): + for filename in filenames: + file_path = os.path.join(root, filename) + if os.path.isfile(file_path): # Skip directories + workbench.upload_files( + scan_code=scan_code, + path=file_path, + chunked_upload=chunked_upload + ) + file_count += 1 + logger.info(f"Uploaded {file_count} files total") + + logger.info("Upload completed successfully.") + + +def extract_archives(workbench: WorkbenchAPI, scan_code: str, recursive: bool = False, jar_extraction: bool = False) -> None: + """ + Extracts archives in the scan. + + Args: + workbench: WorkbenchAPI instance + scan_code: Scan code to extract archives for + recursive: Whether to extract recursively + jar_extraction: Whether to extract JAR files + """ + logger.info("Extracting uploaded archives...") + try: + workbench.extract_archives(scan_code, recursive, jar_extraction) + logger.info("Archive extraction completed.") + except Exception as e: + logger.warning(f"Archive extraction failed: {e}") + logger.info("Continuing with scan process...") + + +def run_kb_scan(workbench: WorkbenchAPI, scan_code: str, scan_options: Dict[str, Any]) -> None: + """ + Runs the knowledge base scan with the provided options. + + Args: + workbench: WorkbenchAPI instance + scan_code: Scan code to run + scan_options: Dictionary of scan configuration options + """ + logger.info("Starting KB scan...") + + try: + workbench.run_scan( + scan_code=scan_code, + limit=scan_options.get("limit", 10), + sensitivity=scan_options.get("sensitivity", 10), + auto_identification_detect_declaration=scan_options.get("auto_identification_detect_declaration", False), + auto_identification_detect_copyright=scan_options.get("auto_identification_detect_copyright", False), + auto_identification_resolve_pending_ids=scan_options.get("auto_identification_resolve_pending_ids", False), + delta_only=scan_options.get("delta_only", False), + reuse_identification=scan_options.get("reuse_identifications", False), + identification_reuse_type=scan_options.get("identification_reuse_type", "any"), + specific_code=scan_options.get("specific_code"), + advanced_match_scoring=scan_options.get("advanced_match_scoring", True), + match_filtering_threshold=scan_options.get("match_filtering_threshold", -1) + ) + logger.info("KB scan started successfully.") + except Exception as e: + raise ProcessError(f"Failed to start KB scan: {e}") + + +def run_dependency_analysis(workbench: WorkbenchAPI, scan_code: str) -> None: + """ + Runs dependency analysis on the scan. + + Args: + workbench: WorkbenchAPI instance + scan_code: Scan code to run dependency analysis on + """ + logger.info("Starting dependency analysis...") + + try: + workbench.start_dependency_analysis(scan_code) + logger.info("Dependency analysis started successfully.") + except Exception as e: + raise ProcessError(f"Failed to start dependency analysis: {e}") + + +def wait_for_scan_completion(workbench: WorkbenchAPI, scan_code: str, timeout_minutes: int) -> None: + """ + Waits for KB scan to complete. + + Args: + workbench: WorkbenchAPI instance + scan_code: Scan code to wait for + timeout_minutes: Maximum time to wait in minutes + """ + logger.info("Waiting for KB scan to complete...") + + try: + # Convert timeout to tries and wait time (using 30 second intervals) + wait_time_seconds = 30 + number_of_tries = (timeout_minutes * 60) // wait_time_seconds + + # Use the new process waiters which return (status_data, duration) + status_data, duration = workbench.wait_for_scan_to_finish( + scan_type="SCAN", + scan_code=scan_code, + scan_number_of_tries=number_of_tries, + scan_wait_time=wait_time_seconds + ) + logger.info("KB scan completed successfully.") + except ProcessTimeoutError: + raise ProcessTimeoutError(f"KB scan timed out after {timeout_minutes} minutes") + except Exception as e: + raise ProcessError(f"KB scan failed: {e}") + + +def wait_for_scan_completion_with_duration(workbench: WorkbenchAPI, scan_code: str, timeout_minutes: int) -> Tuple[bool, float]: + """ + Enhanced version that waits for KB scan to complete and returns duration. + + Args: + workbench: WorkbenchAPI instance + scan_code: Scan code to wait for + timeout_minutes: Maximum time to wait in minutes + + Returns: + Tuple of (success, duration_in_seconds) + """ + start_time = time.time() + + try: + # Convert timeout to tries and wait time (using 30 second intervals) + wait_time_seconds = 30 + number_of_tries = (timeout_minutes * 60) // wait_time_seconds + + # Use the new process waiters which return (status_data, duration) + status_data, duration = workbench.wait_for_scan_to_finish( + scan_type="SCAN", + scan_code=scan_code, + scan_number_of_tries=number_of_tries, + scan_wait_time=wait_time_seconds + ) + logger.info("KB scan completed successfully.") + return True, duration + except Exception as e: + duration = time.time() - start_time + logger.error(f"KB scan failed after {format_duration(duration)}: {e}") + raise e + + +def wait_for_dependency_analysis_completion(workbench: WorkbenchAPI, scan_code: str, timeout_minutes: int) -> None: + """ + Waits for dependency analysis to complete. + + Args: + workbench: WorkbenchAPI instance + scan_code: Scan code to wait for + timeout_minutes: Maximum time to wait in minutes + """ + logger.info("Waiting for dependency analysis to complete...") + + try: + # Convert timeout to tries and wait time (using 30 second intervals) + wait_time_seconds = 30 + number_of_tries = (timeout_minutes * 60) // wait_time_seconds + + # Use the new process waiters which return (status_data, duration) + status_data, duration = workbench.wait_for_scan_to_finish( + scan_type="DEPENDENCY_ANALYSIS", + scan_code=scan_code, + scan_number_of_tries=number_of_tries, + scan_wait_time=wait_time_seconds + ) + logger.info("Dependency analysis completed successfully.") + except ProcessTimeoutError: + raise ProcessTimeoutError(f"Dependency analysis timed out after {timeout_minutes} minutes") + except Exception as e: + raise ProcessError(f"Dependency analysis failed: {e}") + + +def wait_for_dependency_analysis_completion_with_duration(workbench: WorkbenchAPI, scan_code: str, timeout_minutes: int) -> Tuple[bool, float]: + """ + Enhanced version that waits for dependency analysis to complete and returns duration. + + Args: + workbench: WorkbenchAPI instance + scan_code: Scan code to wait for + timeout_minutes: Maximum time to wait in minutes + + Returns: + Tuple of (success, duration_in_seconds) + """ + start_time = time.time() + + try: + # Convert timeout to tries and wait time (using 30 second intervals) + wait_time_seconds = 30 + number_of_tries = (timeout_minutes * 60) // wait_time_seconds + + # Use the new process waiters which return (status_data, duration) + status_data, duration = workbench.wait_for_scan_to_finish( + scan_type="DEPENDENCY_ANALYSIS", + scan_code=scan_code, + scan_number_of_tries=number_of_tries, + scan_wait_time=wait_time_seconds + ) + logger.info("Dependency analysis completed successfully.") + return True, duration + except Exception as e: + duration = time.time() - start_time + logger.error(f"Dependency analysis failed after {format_duration(duration)}: {e}") + raise e + + +def wait_for_archive_extraction(workbench: WorkbenchAPI, scan_code: str, timeout_minutes: int) -> None: + """ + Waits for archive extraction to complete using a 3-second interval. + + Args: + workbench: WorkbenchAPI instance + scan_code: Scan code to wait for + timeout_minutes: Maximum time to wait in minutes + """ + logger.info("Waiting for archive extraction to complete...") + + try: + # Convert timeout to tries using 3-second intervals + wait_time_seconds = 3 # Fixed 3-second interval for archive extraction + number_of_tries = (timeout_minutes * 60) // wait_time_seconds + + # Use the specialized archive extraction waiter + status_data, duration = workbench.wait_for_archive_extraction( + scan_code=scan_code, + scan_number_of_tries=number_of_tries, + scan_wait_time=wait_time_seconds + ) + logger.info("Archive extraction completed successfully.") + except ProcessTimeoutError: + raise ProcessTimeoutError(f"Archive extraction timed out after {timeout_minutes} minutes") + except Exception as e: + raise ProcessError(f"Archive extraction failed: {e}") + + +def wait_for_archive_extraction_with_duration(workbench: WorkbenchAPI, scan_code: str, timeout_minutes: int) -> Tuple[bool, float]: + """ + Enhanced version that waits for archive extraction to complete and returns duration using a 3-second interval. + + Args: + workbench: WorkbenchAPI instance + scan_code: Scan code to wait for + timeout_minutes: Maximum time to wait in minutes + + Returns: + Tuple of (success, duration_in_seconds) + """ + start_time = time.time() + + try: + # Convert timeout to tries using 3-second intervals + wait_time_seconds = 3 # Fixed 3-second interval for archive extraction + number_of_tries = (timeout_minutes * 60) // wait_time_seconds + + # Use the specialized archive extraction waiter + status_data, duration = workbench.wait_for_archive_extraction( + scan_code=scan_code, + scan_number_of_tries=number_of_tries, + scan_wait_time=wait_time_seconds + ) + logger.info("Archive extraction completed successfully.") + return True, duration + except Exception as e: + duration = time.time() - start_time + logger.error(f"Archive extraction failed after {format_duration(duration)}: {e}") + raise e + + +def collect_and_save_results(workbench: WorkbenchAPI, scan_code: str, args) -> Dict[str, Any]: + """ + Collects scan results and saves them if requested. + + Args: + workbench: WorkbenchAPI instance + scan_code: Scan code to get results for + args: Command line arguments + + Returns: + Dictionary containing the collected results + """ + logger.info("Retrieving final scan results...") + + try: + # Get identified licenses (default result type) + results = workbench.get_scan_identified_licenses(scan_code) + + # Save results if path specified + if hasattr(args, 'path_result') and args.path_result: + save_results(args, results) + + return results + + except Exception as e: + logger.error(f"Failed to retrieve results: {e}") + return {} + + +def collect_and_save_results_enhanced(workbench: WorkbenchAPI, scan_code: str, args) -> Dict[str, Any]: + """ + Enhanced result collection with better structure, metadata, and support for different result types. + + Args: + workbench: WorkbenchAPI instance + scan_code: Scan code to get results for + args: Command line arguments + + Returns: + Dictionary containing the collected results with metadata + """ + logger.info("Retrieving final scan results...") + + collected_results = {} + + try: + # Determine what to collect based on existing flags (preserving original functionality) + if getattr(args, 'get_scan_identified_components', False): + results = workbench.get_scan_identified_components(scan_code) + result_type = 'identified_components' + print("Identified components:") + elif getattr(args, 'scans_get_policy_warnings_counter', False): + results = workbench.scans_get_policy_warnings_counter(scan_code) + result_type = 'policy_warnings_counter' + print(f"Scan: {scan_code} policy warnings info:") + elif getattr(args, 'projects_get_policy_warnings_info', False): + results = workbench.projects_get_policy_warnings_info(args.project_code) + result_type = 'project_policy_warnings' + print(f"Project {args.project_code} policy warnings info:") + elif getattr(args, 'scans_get_results', False): + results = workbench.get_results(scan_code) + result_type = 'scan_results' + print(f"Scan {scan_code} results:") + else: + # Default: get identified licenses (original behavior) + results = workbench.get_scan_identified_licenses(scan_code) + result_type = 'identified_licenses' + print("Identified licenses:") + + # Structure the results with metadata + collected_results = { + 'data': results, + 'metadata': { + 'scan_code': scan_code, + 'result_type': result_type, + 'timestamp': time.time(), + 'count': len(results) if isinstance(results, (list, dict)) else 0 + } + } + + # Print the results (preserving original behavior) + import json + print(json.dumps(results)) + + # Save results if path specified + if hasattr(args, 'path_result') and args.path_result: + save_results(args, collected_results) + + return collected_results + + except Exception as e: + logger.error(f"Failed to retrieve results: {e}") + return { + 'data': {}, + 'metadata': { + 'scan_code': scan_code, + 'result_type': 'error', + 'timestamp': time.time(), + 'count': 0, + 'error': str(e) + } + } + + +def cleanup_temp_file(file_path: str) -> bool: + """ + Cleanup temporary file safely. + + Args: + file_path: Path to temporary file to clean up + + Returns: + True if cleanup was successful, False otherwise + """ + try: + if os.path.exists(file_path): + os.remove(file_path) + logger.debug(f"Cleaned up temporary file: {file_path}") + return True + return True # File doesn't exist, consider it cleaned up + except Exception as e: + logger.warning(f"Failed to clean up temporary file {file_path}: {e}") + return False + + +def format_duration(duration_seconds: Optional[Union[int, float]]) -> str: + """ + Formats a duration in seconds into a human-readable string. + + Args: + duration_seconds: Duration in seconds + + Returns: + Formatted duration string + """ + if duration_seconds is None: + return "N/A" + try: + duration_seconds = round(float(duration_seconds)) + except (ValueError, TypeError): + return "Invalid Duration" + + minutes, seconds = divmod(int(duration_seconds), 60) + if minutes > 0 and seconds > 0: + return f"{minutes} minutes, {seconds} seconds" + elif minutes > 0: + return f"{minutes} minutes" + elif seconds == 1: + return f"1 second" + else: + return f"{seconds} seconds" + + +def print_operation_summary(params, scan_completed: bool, da_completed: bool, + project_code: str, scan_code: str, durations: Dict[str, float] = None) -> None: + """ + Prints a standardized summary of the scan operations performed and settings used. + + Args: + params: Command line parameters + scan_completed: Whether KB scan completed successfully + da_completed: Whether dependency analysis completed successfully + project_code: Project code associated with the scan + scan_code: Scan code of the operation + durations: Dictionary containing operation durations in seconds + """ + durations = durations or {} + + print(f"\n--- Operation Summary ---") + print("Workbench Agent Operation Details:") + + # Determine scan method + if getattr(params, 'blind_scan', False) or getattr(params, 'scan_type', None) == 'blind_scan': + print(f" - Method: Blind Scan (using CLI hash generation)") + else: + print(f" - Method: Code Upload (using --path)") + + print(f" - Source Path: {getattr(params, 'path', 'N/A')}") + print(f" - Recursive Archive Extraction: {getattr(params, 'recursively_extract_archives', 'N/A')}") + print(f" - JAR File Extraction: {getattr(params, 'jar_file_extraction', 'N/A')}") + + print("\nScan Parameters:") + print(f" - Auto-ID File Licenses: {'Yes' if getattr(params, 'auto_identification_detect_declaration', False) else 'No'}") + print(f" - Auto-ID File Copyrights: {'Yes' if getattr(params, 'auto_identification_detect_copyright', False) else 'No'}") + print(f" - Auto-ID Pending IDs: {'Yes' if getattr(params, 'auto_identification_resolve_pending_ids', False) else 'No'}") + print(f" - Delta Scan: {'Yes' if getattr(params, 'delta_only', False) else 'No'}") + print(f" - Identification Reuse: {'Yes' if getattr(params, 'reuse_identifications', False) else 'No'}") + + print("\nAnalysis Performed:") + kb_scan_performed = not getattr(params, 'run_only_dependency_analysis', False) + + if kb_scan_performed and scan_completed: + kb_duration_str = format_duration(durations.get("kb_scan", 0)) if durations.get("kb_scan") else "N/A" + print(f" - Signature Scan: Yes (Duration: {kb_duration_str})") + elif kb_scan_performed: + print(f" - Signature Scan: Started but not completed") + else: + print(f" - Signature Scan: No") + + if da_completed: + da_duration_str = format_duration(durations.get("dependency_analysis", 0)) if durations.get("dependency_analysis") else "N/A" + print(f" - Dependency Analysis: Yes (Duration: {da_duration_str})") + else: + print(f" - Dependency Analysis: No") + + # Show extraction duration if available + if durations.get("extraction_duration", 0) > 0: + extraction_duration_str = format_duration(durations.get("extraction_duration", 0)) + print(f" - Archive Extraction: Yes (Duration: {extraction_duration_str})") + + print("------------------------------------") \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..6a39a4f --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +# Tests package for workbench-agent API modules diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 0000000..cefb91a --- /dev/null +++ b/tests/unit/__init__.py @@ -0,0 +1 @@ +# Unit tests for workbench-agent diff --git a/tests/unit/api/__init__.py b/tests/unit/api/__init__.py new file mode 100644 index 0000000..4d1abef --- /dev/null +++ b/tests/unit/api/__init__.py @@ -0,0 +1 @@ +# Unit tests for API modules diff --git a/tests/unit/api/test_projects_api.py b/tests/unit/api/test_projects_api.py new file mode 100644 index 0000000..a6a88c7 --- /dev/null +++ b/tests/unit/api/test_projects_api.py @@ -0,0 +1,122 @@ +# tests/unit/api/test_projects_api.py + +import pytest +import json +import builtins +from unittest.mock import MagicMock, patch, Mock + +# Import from our API structure +from workbench_agent.api.projects_api import ProjectsAPI + + +# --- Fixtures --- +@pytest.fixture +def projects_api_inst(): + """Create a ProjectsAPI instance for testing.""" + return ProjectsAPI( + api_url="http://dummy.com/api.php", api_user="testuser", api_token="testtoken" + ) + + +# --- Test Cases --- + + + + + +# --- Test create_project --- +@patch.object(ProjectsAPI, "_send_request") +@patch("builtins.print") # Mock print to avoid output during tests +def test_create_project_success(mock_print, mock_send, projects_api_inst): + """Test successful project creation.""" + mock_send.return_value = {"status": "1", "data": {"project_id": 123}} + + # Should not raise exception + projects_api_inst.create_project("new_project") + + mock_send.assert_called_once() + call_args = mock_send.call_args[0][0] + assert call_args["group"] == "projects" + assert call_args["action"] == "create" + assert call_args["data"]["project_code"] == "new_project" + assert call_args["data"]["project_name"] == "new_project" + assert "Automatically created by Workbench Agent script" in call_args["data"]["description"] + + # Check print was called with success message + mock_print.assert_called_once_with("Created project new_project") + + +@patch.object(ProjectsAPI, "_send_request") +def test_create_project_failure(mock_send, projects_api_inst): + """Test project creation failure.""" + mock_send.return_value = {"status": "0", "error": "Project already exists"} + + with pytest.raises(builtins.Exception) as exc_info: + projects_api_inst.create_project("existing_project") + + assert "Failed to create project" in str(exc_info.value) + mock_send.assert_called_once() + + +# --- Test projects_get_policy_warnings_info --- +@patch.object(ProjectsAPI, "_send_request") +def test_projects_get_policy_warnings_info_success(mock_send, projects_api_inst): + """Test successful retrieval of project policy warnings.""" + mock_response = { + "status": "1", + "data": { + "warnings_count": 5, + "warnings": [ + {"type": "license", "severity": "high", "message": "GPL license detected"} + ], + }, + } + mock_send.return_value = mock_response + + result = projects_api_inst.projects_get_policy_warnings_info("test_project") + + assert result == mock_response["data"] + mock_send.assert_called_once() + call_args = mock_send.call_args[0][0] + assert call_args["group"] == "projects" + assert call_args["action"] == "get_policy_warnings_info" + assert call_args["data"]["project_code"] == "test_project" + + +@patch.object(ProjectsAPI, "_send_request") +def test_projects_get_policy_warnings_info_failure(mock_send, projects_api_inst): + """Test policy warnings retrieval failure.""" + from workbench_agent.exceptions import ApiError + mock_send.return_value = {"status": "0", "error": "Project not found"} + + with pytest.raises(ApiError) as exc_info: + projects_api_inst.projects_get_policy_warnings_info("nonexistent_project") + + assert "Failed to get policy warnings info" in str(exc_info.value) + + +@patch.object(ProjectsAPI, "_send_request") +def test_projects_get_policy_warnings_info_no_data(mock_send, projects_api_inst): + """Test policy warnings retrieval when no data key in response.""" + from workbench_agent.exceptions import ApiError + mock_send.return_value = {"status": "1"} # No "data" key + + with pytest.raises(ApiError) as exc_info: + projects_api_inst.projects_get_policy_warnings_info("test_project") + + assert "Failed to get policy warnings info" in str(exc_info.value) + + +# --- Test API Base Integration --- +def test_projects_api_inherits_from_api_base(projects_api_inst): + """Test that ProjectsAPI properly inherits from APIBase.""" + # Check that it has the required attributes from APIBase + assert hasattr(projects_api_inst, "api_url") + assert hasattr(projects_api_inst, "api_user") + assert hasattr(projects_api_inst, "api_token") + assert hasattr(projects_api_inst, "_send_request") + + # Check that attributes are set correctly + assert projects_api_inst.api_url == "http://dummy.com/api.php" + assert projects_api_inst.api_user == "testuser" + assert projects_api_inst.api_token == "testtoken" diff --git a/tests/unit/api/test_scans_api.py b/tests/unit/api/test_scans_api.py new file mode 100644 index 0000000..0afc5ec --- /dev/null +++ b/tests/unit/api/test_scans_api.py @@ -0,0 +1,254 @@ +# tests/unit/api/test_scans_api.py + +import pytest +import json +import builtins +import time +from unittest.mock import MagicMock, patch, Mock + +# Import from our API structure +from workbench_agent.api.scans_api import ScansAPI + + +# --- Fixtures --- +@pytest.fixture +def scans_api_inst(): + """Create a ScansAPI instance for testing.""" + return ScansAPI(api_url="http://dummy.com/api.php", api_user="testuser", api_token="testtoken") + + +# --- Test Cases --- + + +# --- Test create_webapp_scan --- +@patch.object(ScansAPI, "_send_request") +def test_create_webapp_scan_success(mock_send, scans_api_inst): + """Test successful scan creation.""" + mock_send.return_value = {"status": "1", "data": {"scan_id": 999}} + + result = scans_api_inst.create_webapp_scan("test_scan", "test_project") + + assert result == 999 + mock_send.assert_called_once() + call_args = mock_send.call_args[0][0] + assert call_args["group"] == "scans" + assert call_args["action"] == "create" + assert call_args["data"]["scan_code"] == "test_scan" + assert call_args["data"]["scan_name"] == "test_scan" + assert call_args["data"]["project_code"] == "test_project" + + +@patch.object(ScansAPI, "_send_request") +def test_create_webapp_scan_failure(mock_send, scans_api_inst): + """Test scan creation failure.""" + mock_send.return_value = {"status": "0", "error": "Scan already exists"} + + with pytest.raises(builtins.Exception) as exc_info: + scans_api_inst.create_webapp_scan("existing_scan", "test_project") + + assert "Failed to create scan" in str(exc_info.value) + + + + + + + +# --- Test get_scan_status --- +@patch.object(ScansAPI, "_send_request") +def test_get_scan_status_success(mock_send, scans_api_inst): + """Test successful scan status retrieval.""" + mock_response = { + "status": "1", + "data": {"is_finished": "0", "percentage_done": "75%", "status": "RUNNING"}, + } + mock_send.return_value = mock_response + + result = scans_api_inst.get_scan_status("SCAN", "test_scan") + + assert result == mock_response["data"] + call_args = mock_send.call_args[0][0] + assert call_args["group"] == "scans" + assert call_args["action"] == "check_status" + assert call_args["data"]["scan_code"] == "test_scan" + assert call_args["data"]["type"] == "SCAN" + + +@patch.object(ScansAPI, "_send_request") +def test_get_scan_status_failure(mock_send, scans_api_inst): + """Test scan status retrieval failure.""" + from workbench_agent.exceptions import ScanNotFoundError + mock_send.return_value = {"status": "0", "error": "Scan not found"} + + with pytest.raises(ScanNotFoundError): + scans_api_inst.get_scan_status("SCAN", "nonexistent_scan") + + +# --- Test start_dependency_analysis --- +@patch.object(ScansAPI, "assert_dependency_analysis_can_start") +@patch.object(ScansAPI, "_send_request") +def test_start_dependency_analysis_success(mock_send, mock_assert, scans_api_inst): + """Test successful dependency analysis start.""" + mock_send.return_value = {"status": "1", "data": {"message": "Started"}} + mock_assert.return_value = None # No exception means can start + + # Should not raise exception + scans_api_inst.start_dependency_analysis("test_scan") + + mock_assert.assert_called_once_with("test_scan") + mock_send.assert_called_once() + call_args = mock_send.call_args[0][0] + assert call_args["group"] == "scans" + assert call_args["action"] == "run_dependency_analysis" + assert call_args["data"]["scan_code"] == "test_scan" + + +@patch.object(ScansAPI, "_send_request") +def test_start_dependency_analysis_failure(mock_send, scans_api_inst): + """Test dependency analysis start failure.""" + mock_send.return_value = {"status": "0", "error": "Cannot start analysis"} + + with pytest.raises(builtins.Exception) as exc_info: + scans_api_inst.start_dependency_analysis("test_scan") + + assert "Failed to start dependency analysis" in str(exc_info.value) + + +# --- Test wait_for_scan_to_finish --- +@patch.object(ScansAPI, "check_status") +@patch("time.sleep") # Mock sleep to speed up tests +@patch("builtins.print") # Mock print to avoid output during tests +def test_wait_for_scan_to_finish_success(mock_print, mock_sleep, mock_get_status, scans_api_inst): + """Test successful scan completion waiting.""" + # Mock scan progression: running -> finished + # Note: is_finished="0" means not finished, is_finished="1" or "FINISHED" status means finished + mock_get_status.side_effect = [ + {"is_finished": False, "percentage_done": "50%", "status": "RUNNING"}, + {"is_finished": "1", "percentage_done": "100%", "status": "FINISHED"}, + ] + + result = scans_api_inst.wait_for_scan_to_finish("SCAN", "test_scan", 5, 1) + + # The new implementation returns a tuple (status_data, duration) + assert isinstance(result, tuple) + status_data, duration = result + assert status_data["status"] == "FINISHED" + assert status_data["percentage_done"] == "100%" + assert isinstance(duration, float) + assert mock_get_status.call_count == 2 + mock_sleep.assert_called_once_with(1) + + +@patch.object(ScansAPI, "check_status") +@patch("time.sleep") +@patch("builtins.print") +def test_wait_for_scan_to_finish_timeout(mock_print, mock_sleep, mock_get_status, scans_api_inst): + """Test scan waiting timeout.""" + from workbench_agent.exceptions import ProcessTimeoutError + # Mock scan always running - is_finished=False means not finished + mock_get_status.return_value = { + "is_finished": False, + "percentage_done": "50%", + "status": "RUNNING", + } + + with pytest.raises(ProcessTimeoutError) as exc_info: + scans_api_inst.wait_for_scan_to_finish("SCAN", "test_scan", 2, 1) + + assert "Timeout waiting for" in str(exc_info.value) + assert mock_get_status.call_count == 2 + + +# --- Test get_scan_identified_licenses --- +@patch.object(ScansAPI, "_send_request") +def test_get_scan_identified_licenses_success(mock_send, scans_api_inst): + """Test successful license retrieval.""" + mock_response = { + "status": "1", + "data": [{"license": "MIT", "count": 5}, {"license": "GPL-3.0", "count": 2}], + } + mock_send.return_value = mock_response + + result = scans_api_inst.get_scan_identified_licenses("test_scan") + + assert result == mock_response["data"] + call_args = mock_send.call_args[0][0] + assert call_args["group"] == "scans" + assert call_args["action"] == "get_scan_identified_licenses" + assert call_args["data"]["unique"] == "1" + + +# --- Test extract_archives --- +@patch.object(ScansAPI, "_send_request") +def test_extract_archives_success(mock_send, scans_api_inst): + """Test successful archive extraction.""" + mock_send.return_value = {"status": "1", "data": {"message": "Extracted"}} + + result = scans_api_inst.extract_archives("test_scan", True, False) + + assert result is True + call_args = mock_send.call_args[0][0] + assert call_args["group"] == "scans" + assert call_args["action"] == "extract_archives" + assert call_args["data"]["recursively_extract_archives"] is True + assert call_args["data"]["jar_file_extraction"] is False + + +@patch.object(ScansAPI, "_send_request") +def test_extract_archives_failure(mock_send, scans_api_inst): + """Test archive extraction failure.""" + from workbench_agent.exceptions import ApiError + mock_send.return_value = {"status": "0", "error": "Cannot extract"} + + with pytest.raises(ApiError) as exc_info: + scans_api_inst.extract_archives("test_scan", True, False) + + assert "Failed to extract archives" in str(exc_info.value) + assert "Cannot extract" in str(exc_info.value) + + +# --- Test remove_uploaded_content --- +@patch.object(ScansAPI, "_send_request") +@patch("builtins.print") +def test_remove_uploaded_content_success(mock_print, mock_send, scans_api_inst): + """Test successful file content removal.""" + mock_send.return_value = {"status": "1", "data": {"message": "Removed"}} + + scans_api_inst.remove_uploaded_content("test.zip", "test_scan") + + call_args = mock_send.call_args[0][0] + assert call_args["group"] == "scans" + assert call_args["action"] == "remove_uploaded_content" + assert call_args["data"]["filename"] == "test.zip" + assert call_args["data"]["scan_code"] == "test_scan" + + # Should print the action being taken + assert mock_print.call_count >= 1 + + +@patch.object(ScansAPI, "_send_request") +@patch("builtins.print") +def test_remove_uploaded_content_failure(mock_print, mock_send, scans_api_inst): + """Test file content removal failure.""" + mock_send.return_value = {"status": "0", "error": "File not found"} + + # Should not raise exception, just print warning + scans_api_inst.remove_uploaded_content("test.zip", "test_scan") + + # Should print warning about failure + assert any("Cannot delete file" in str(call) for call in mock_print.call_args_list) + + +# --- Test API Base Integration --- +def test_scans_api_inherits_from_api_base(scans_api_inst): + """Test that ScansAPI properly inherits from APIBase.""" + # Check that it has the required attributes from APIBase + assert hasattr(scans_api_inst, "api_url") + assert hasattr(scans_api_inst, "api_user") + assert hasattr(scans_api_inst, "api_token") + assert hasattr(scans_api_inst, "_send_request") + + # Check that attributes are set correctly + assert scans_api_inst.api_url == "http://dummy.com/api.php" + assert scans_api_inst.api_user == "testuser" + assert scans_api_inst.api_token == "testtoken" diff --git a/tests/unit/api/test_workbench_api.py b/tests/unit/api/test_workbench_api.py new file mode 100644 index 0000000..7c90012 --- /dev/null +++ b/tests/unit/api/test_workbench_api.py @@ -0,0 +1,176 @@ +# tests/unit/api/test_workbench_api.py + +import pytest +import json +import builtins +from unittest.mock import MagicMock, patch, Mock + +# Import from our API structure +from workbench_agent.api.workbench_api import WorkbenchAPI + + +# --- Fixtures --- +@pytest.fixture +def workbench_inst(): + """Create a WorkbenchAPI instance for testing.""" + return WorkbenchAPI( + api_url="http://dummy.com/api.php", api_user="testuser", api_token="testtoken" + ) + + +# --- Test Cases --- + + +def test_workbench_api_composition(workbench_inst): + """Test that WorkbenchAPI properly composes all API modules.""" + # Check that it has methods from all API classes + + # ProjectsAPI methods + assert hasattr(workbench_inst, "create_project") + assert hasattr(workbench_inst, "projects_get_policy_warnings_info") + + # ScansAPI methods + assert hasattr(workbench_inst, "create_webapp_scan") + assert hasattr(workbench_inst, "run_scan") + assert hasattr(workbench_inst, "start_dependency_analysis") + assert hasattr(workbench_inst, "wait_for_scan_to_finish") + assert hasattr(workbench_inst, "get_scan_identified_licenses") + assert hasattr(workbench_inst, "remove_uploaded_content") # Moved from UploadAPI + + # UploadAPI methods + assert hasattr(workbench_inst, "upload_files") + + +def test_workbench_api_inheritance_order(workbench_inst): + """Test that the method resolution order is correct.""" + # Check MRO includes all expected classes + mro_names = [cls.__name__ for cls in type(workbench_inst).__mro__] + + expected_classes = [ + "WorkbenchAPI", + "ProjectsAPI", + "ScansAPI", + "UploadAPI", + "UploadHelper", + "APIBase", + ] + + for expected_class in expected_classes: + assert expected_class in mro_names, f"{expected_class} not found in MRO" + + +def test_workbench_api_inherits_from_api_base(workbench_inst): + """Test that WorkbenchAPI properly inherits from APIBase.""" + # Check that it has the required attributes from APIBase + assert hasattr(workbench_inst, "api_url") + assert hasattr(workbench_inst, "api_user") + assert hasattr(workbench_inst, "api_token") + assert hasattr(workbench_inst, "_send_request") + + # Check that attributes are set correctly + assert workbench_inst.api_url == "http://dummy.com/api.php" + assert workbench_inst.api_user == "testuser" + assert workbench_inst.api_token == "testtoken" + + +# --- Integration Tests --- +@patch.object(WorkbenchAPI, "_send_request") +def test_workbench_api_project_workflow(mock_send, workbench_inst): + """Test a typical project workflow using the composed API.""" + # Mock responses for a typical workflow + mock_send.side_effect = [ + {"status": "1", "data": {"project_id": 123}}, # create_project + {"status": "1", "data": {"scan_id": 999}}, # create_webapp_scan + ] + + # Execute simplified workflow (using try/catch pattern) + workbench_inst.create_project("test_project") # Should not raise + + scan_id = workbench_inst.create_webapp_scan("test_scan", "test_project") + assert scan_id == 999 + + # Verify all calls were made + assert mock_send.call_count == 2 + + +@patch.object(WorkbenchAPI, "_send_request") +def test_workbench_api_scan_workflow(mock_send, workbench_inst): + """Test a typical scan workflow using the composed API.""" + # Mock responses for scan operations + mock_send.side_effect = [ + {"status": "1", "data": {"scan_id": 999}}, # create_webapp_scan + {"status": "1", "data": {"message": "Extracted"}}, # extract_archives + {"status": "1", "data": {"message": "Started"}}, # run_scan + {"status": "1", "data": [{"license": "MIT", "count": 5}]}, # get_scan_identified_licenses + ] + + # Execute scan workflow + scan_id = workbench_inst.create_webapp_scan("test_scan", "test_project") + assert scan_id == 999 + + extract_result = workbench_inst.extract_archives("test_scan", True, False) + assert extract_result is True + + # Note: run_scan has many parameters, using minimal set for test + with patch.object(workbench_inst, "assert_scan_can_start"): + run_result = workbench_inst.run_scan("test_scan", 10, 10, False, False, False, False, False) + assert run_result["status"] == "1" + + licenses = workbench_inst.get_scan_identified_licenses("test_scan") + assert len(licenses) == 1 + assert licenses[0]["license"] == "MIT" + + + +# --- Error Handling Integration Tests --- +@patch.object(WorkbenchAPI, "_send_request") +def test_workbench_api_error_propagation(mock_send, workbench_inst): + """Test that errors are properly propagated through the composed API.""" + mock_send.return_value = {"status": "0", "error": "API Error"} + + # Test that errors from different API modules are handled consistently + with pytest.raises(builtins.Exception): + workbench_inst.create_project("test_project") + + with pytest.raises(builtins.Exception): + workbench_inst.create_webapp_scan("test_scan", "test_project") + + with pytest.raises(builtins.Exception): + workbench_inst.start_dependency_analysis("test_scan") + + +# --- Method Resolution Tests --- +def test_remove_uploaded_content_comes_from_scans_api(workbench_inst): + """Test that remove_uploaded_content method comes from ScansAPI, not UploadAPI.""" + # Find which class in the MRO defines remove_uploaded_content + mro = type(workbench_inst).__mro__ + defining_class = None + + for cls in mro: + if hasattr(cls, "remove_uploaded_content") and "remove_uploaded_content" in cls.__dict__: + defining_class = cls + break + + # Should be defined in ScansAPI, not UploadAPI + assert defining_class.__name__ == "ScansAPI" + + +# --- Backwards Compatibility Test --- +def test_workbench_alias_compatibility(): + """Test that the Workbench alias still works for backwards compatibility.""" + # Import the alias from the main module + import sys + import os + + sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../..")) + + from workbench_agent.api import WorkbenchAPI + + # The alias should be the same class + assert WorkbenchAPI is not None + + # Create instance using the main class + wb = WorkbenchAPI("http://test.com/api.php", "user", "token") + assert wb.api_url == "http://test.com/api.php" + assert wb.api_user == "user" + assert wb.api_token == "token" diff --git a/workbench-agent.py b/workbench-agent.py old mode 100755 new mode 100644 index f02cb64..755a210 --- a/workbench-agent.py +++ b/workbench-agent.py @@ -2,1523 +2,13 @@ # Copyright: FossID AB 2022 -import builtins -import json -import time -import logging -import argparse -import random -import base64 -import io -import os -import subprocess -from argparse import RawTextHelpFormatter import sys -import traceback -import requests +from workbench_agent.main import main # Import the main function from the package -# from dotenv import load_dotenv -logger = logging.getLogger("log") +# Keep backward compatibility by creating an alias +from workbench_agent.api import WorkbenchAPI +Workbench = WorkbenchAPI -class Workbench: - """ - A class to interact with the FossID Workbench API for managing scans and projects. - - Attributes: - api_url (str): The base URL of the Workbench API. - api_user (str): The username used for API authentication. - api_token (str): The API token for authentication. - """ - - def __init__(self, api_url: str, api_user: str, api_token: str): - """ - Initializes the Workbench object with API credentials and endpoint. - - Args: - api_url (str): The base URL of the Workbench API. - api_user (str): The username used for API authentication. - api_token (str): The API token for authentication. - """ - self.api_url = api_url - self.api_user = api_user - self.api_token = api_token - - def _send_request(self, payload: dict) -> dict: - """ - Sends a request to the Workbench API. - - Args: - payload (dict): The payload of the request. - - Returns: - dict: The JSON response from the API. - """ - url = self.api_url - headers = { - "Accept": "*/*", - "Content-Type": "application/json; charset=utf-8", - } - req_body = json.dumps(payload) - logger.debug("url %s", url) - logger.debug("url %s", headers) - logger.debug(req_body) - response = requests.request( - "POST", url, headers=headers, data=req_body, timeout=1800 - ) - logger.debug(response.text) - try: - # Attempt to parse the JSON - parsed_json = json.loads(response.text) - return parsed_json - except json.JSONDecodeError as e: - # If an error occurs, catch it and display the message along with the problematic JSON - print("Failed to decode JSON") - print(f"Error message: {e.msg}") - print(f"At position: {e.pos}") - print("Problematic JSON:") - print(response.text) - - def _read_in_chunks(self,file_object: io.BufferedReader, chunk_size=5242880): - """ - Generator to read a file piece by piece. - - Args: - file_object (io.BufferedReader) : The payload of the request. - chunk_size (int): Size of the chunk. Default chunk size is 5MB - """ - while True: - data = file_object.read(chunk_size) - if not data: - break - yield data - - def _chunked_upload_request(self, scan_code: str, headers: dict, chunk: bytes): - """ - This function will make sure Content-Length header is not sent by Requests library - Args: - scan_code (str): The scan code where the file or files will be uploaded. - headers (dict) : Headers for HTTP request - chunk (bytes): Chunk read from large file - """ - try: - req = requests.Request( - 'POST', - self.api_url, - headers=headers, - data=chunk, - auth=(self.api_user, self.api_token), - ) - s = requests.Session() - prepped = s.prepare_request(req) - # Remove the unwanted header 'Content-Length' !!! - if 'Content-Length' in prepped.headers: - del prepped.headers['Content-Length'] - - # Send HTTP request and retrieve response - response = s.send(prepped) - # print(f"Sent headers: {response.request.headers}") - # print(f"response headers: {response.headers}") - # Retrieve the HTTP status code - status_code = response.status_code - print(f"HTTP Status Code: {status_code}") - - # Check if the request was successful (status code 200) - if status_code == 200: - # Parse the JSON response - try: - response.json() - except: - print(f"Failed to decode json {response.text}") - print(traceback.print_exc()) - sys.exit(1) - else: - print(f"Request failed with status code {status_code}") - reason = response.reason - print(f"Reason: {reason}") - response_text = response.text - print(f"Response Text: {response_text}") - sys.exit(1) - except IOError: - # Error opening file - print(f"Failed to upload files to the scan {scan_code}.") - print(traceback.print_exc()) - sys.exit(1) - - def upload_files(self, scan_code: str, path: str, chunked_upload: bool = False): - """ - Uploads files to the Workbench using the API's File Upload endpoint. - - Args: - scan_code (str): The scan code where the file or files will be uploaded. - path (str): Path to the file or files to upload. - chunked_upload (bool): Enable/disable chunk upload. - """ - file_size = os.path.getsize(path) - size_limit = 8 * 1024 * 1024 # 8MB in bytes. Based on the default value of post_max_size in php.ini - # Prepare parameters - filename = os.path.basename(path) - filename_base64 = base64.b64encode(filename.encode()).decode("utf-8") - scan_code_base64 = base64.b64encode(scan_code.encode()).decode("utf-8") - - if chunked_upload and (file_size > size_limit): - print(f"Uploading {filename} using 'Transfer-encoding: chunks' due to file size {file_size}.") - # Use chunked upload for files bigger than size_limit - # First delete possible existing files because chunk uploading works by appending existing file on disk. - self.remove_uploaded_content(filename, scan_code) - print("Uploading using Transfer-encoding: chunked...") - headers = { - "FOSSID-SCAN-CODE": scan_code_base64, - "FOSSID-FILE-NAME": filename_base64, - 'Transfer-Encoding': 'chunked', - 'Content-Type': 'application/octet-stream' - } - try: - with open(path, "rb") as file: - for chunk in self._read_in_chunks(file, 5242880): - # Upload each chunk - self._chunked_upload_request(scan_code, headers, chunk) - except IOError: - # Error opening file - print(f"Failed to upload files to the scan {scan_code}.") - print(traceback.print_exc()) - sys.exit(1) - print("Finished uploading.") - else: - # Regular upload, no chunk upload - headers = { - "FOSSID-SCAN-CODE": scan_code_base64, - "FOSSID-FILE-NAME": filename_base64 - } - print("Uploading...") - try: - with open(path, "rb") as file: - resp = requests.post( - self.api_url, - headers=headers, - data=file, - auth=(self.api_user, self.api_token), - timeout=1800, - ) - # Retrieve the HTTP status code - status_code = resp.status_code - print(f"HTTP Status Code: {status_code}") - - # Check if the request was successful (status code 200) - if status_code == 200: - # Parse the JSON response - try: - resp.json() - except: - print(f"Failed to decode json {resp.text}") - print(traceback.print_exc()) - sys.exit(1) - else: - print(f"Request failed with status code {status_code}") - reason = resp.reason - print(f"Reason: {reason}") - response_text = resp.text - print(f"Response Text: {response_text}") - sys.exit(1) - except IOError: - # Error opening file - print(f"Failed to upload files to the scan {scan_code}.") - print(traceback.print_exc()) - sys.exit(1) - print("Finished uploading.") - - def _delete_existing_scan(self, scan_code: str): - """ - Deletes a scan - - Args: - scan_code (str): The code of the scan to be deleted - - Returns: - dict: The JSON response from the API. - """ - payload = { - "group": "scans", - "action": "delete", - "data": { - "username": self.api_user, - "key": self.api_token, - "scan_code": scan_code, - "delete_identifications": "true", - }, - } - return self._send_request(payload) - - def create_webapp_scan(self, scan_code: str, project_code: str = None, target_path: str = None) -> bool: - """ - Creates a Scan in Workbench. The scan can optionally be created inside a Project. - - Args: - scan_code (str): The unique identifier for the scan. - project_code (str, optional): The project code within which to create the scan. - target_path (str, optional): The target path where scan is stored. - - Returns: - bool: True if the scan was successfully created, False otherwise. - """ - payload = { - "group": "scans", - "action": "create", - "data": { - "username": self.api_user, - "key": self.api_token, - "scan_code": scan_code, - "scan_name": scan_code, - "project_code": project_code, - "target_path": target_path, - "description": "Scan created using the Workbench Agent.", - }, - } - response = self._send_request(payload) - if response["status"] != "1": - raise builtins.Exception( - "Failed to create scan {}: {}".format(scan_code, response) - ) - if "error" in response.keys(): - raise builtins.Exception( - "Failed to create scan {}: {}".format(scan_code, response["error"]) - ) - return response["data"]["scan_id"] - - def _get_scan_status(self, scan_type: str, scan_code: str): - """ - Calls API scans -> check_status to determine if the process is finished. - - Args: - scan_type (str): One of these: SCAN, REPORT_IMPORT, DEPENDENCY_ANALYSIS, REPORT_GENERATION, DELETE_SCAN. - scan_code (str): The unique identifier for the scan. - - Returns: - dict: The data section from the JSON response returned from API. - """ - payload = { - "group": "scans", - "action": "check_status", - "data": { - "username": self.api_user, - "key": self.api_token, - "scan_code": scan_code, - "type": scan_type, - }, - } - response = self._send_request(payload) - if response["status"] != "1": - raise builtins.Exception( - "Failed to retrieve scan status from \ - scan {}: {}".format( - scan_code, response["error"] - ) - ) - return response["data"] - - def start_dependency_analysis(self, scan_code: str): - """ - Initiate dependency analysis for a scan. - - Args: - scan_code (str): The unique identifier for the scan. - """ - payload = { - "group": "scans", - "action": "run_dependency_analysis", - "data": { - "username": self.api_user, - "key": self.api_token, - "scan_code": scan_code, - }, - } - response = self._send_request(payload) - if response["status"] != "1": - raise builtins.Exception( - "Failed to start dependency analysis scan {}: {}".format( - scan_code, response["error"] - ) - ) - - def wait_for_scan_to_finish( - self, - scan_type: str, - scan_code: str, - scan_number_of_tries: int, - scan_wait_time: int, - ): - """ - Check if the scan finished after each 'scan_wait_time' seconds for 'scan_number_of_tries' number of tries. - If the scan is finished return true. If the scan is not finished after all tries throw Exception. - - Args: - scan_type (str): Types: SCAN, REPORT_IMPORT, DEPENDENCY_ANALYSIS, REPORT_GENERATION, DELETE_SCAN - scan_code (str): Unique scan identifier. - scan_number_of_tries (int): Number of calls to "check_status" till declaring the scan failed. - scan_wait_time (int): Time interval between calling "check_status", expressed in seconds - - Returns: - bool - """ - # pylint: disable-next=unused-variable - for x in range(scan_number_of_tries): - scan_status = self._get_scan_status(scan_type, scan_code) - is_finished = ( - scan_status["is_finished"] - or scan_status["is_finished"] == "1" - or scan_status["status"] == "FAILED" - or scan_status["status"] == "FINISHED" - ) - if is_finished: - if ( - scan_status["percentage_done"] == "100%" - or scan_status["percentage_done"] == 100 - or ( - scan_type == "DEPENDENCY_ANALYSIS" - and ( - scan_status["percentage_done"] == "0%" - or scan_status["percentage_done"] == "0%%" - ) - ) - ): - print( - "Scan percentage_done = 100%, scan has finished. Status: {}".format( - scan_status["status"] - ) - ) - return True - raise builtins.Exception( - "Scan finished with status: {} percentage: {} ".format( - scan_status["status"], scan_status["percentage_done"] - ) - ) - # If scan did not finished, print info about progress - print( - "Scan {} is running. Percentage done: {}% Status: {}".format( - scan_code, scan_status["percentage_done"], scan_status["status"] - ) - ) - # Wait given time - time.sleep(scan_wait_time) - # If this code is reached it means the scan didn't finished after scan_number_of_tries X scan_wait_time - print("{} timeout: {}".format(scan_type, scan_code)) - raise builtins.Exception("scan timeout") - - def _get_pending_files(self, scan_code: str): - """ - Call API scans -> get_pending_files. - - Args: - scan_code (str): The unique identifier for the scan. - - Returns: - dict: The JSON response from the API. - """ - payload = { - "group": "scans", - "action": "get_pending_files", - "data": { - "username": self.api_user, - "key": self.api_token, - "scan_code": scan_code, - }, - } - response = self._send_request(payload) - if response["status"] == "1" and "data" in response.keys(): - return response["data"] - # all other situations - raise builtins.Exception( - "Error getting pending files \ - result: {}".format( - response - ) - ) - - def scans_get_policy_warnings_counter(self, scan_code: str): - """ - Retrieve policy warnings information at scan level. - - Args: - scan_code (str): The unique identifier for the scan. - - Returns: - dict: The JSON response from the API. - """ - payload = { - "group": "scans", - "action": "get_policy_warnings_counter", - "data": { - "username": self.api_user, - "key": self.api_token, - "scan_code": scan_code, - }, - } - response = self._send_request(payload) - if response["status"] == "1" and "data" in response.keys(): - return response["data"] - raise builtins.Exception( - "Error getting project policy warnings information \ - result: {}".format( - response - ) - ) - - def projects_get_policy_warnings_info(self, project_code: str): - """ - Retrieve policy warnings information at project level. - - Args: - project_code (str): The unique identifier for the project. - - Returns: - dict: The JSON response from the API. - """ - payload = { - "group": "projects", - "action": "get_policy_warnings_info", - "data": { - "username": self.api_user, - "key": self.api_token, - "project_code": project_code, - }, - } - response = self._send_request(payload) - if response["status"] == "1" and "data" in response.keys(): - return response["data"] - raise builtins.Exception( - "Error getting project policy warnings information \ - result: {}".format( - response - ) - ) - - def get_scan_identified_components(self, scan_code: str): - """ - Retrieve the list of identified components from one scan. - - Args: - scan_code (str): The unique identifier for the scan. - - Returns: - dict: The JSON response from the API. - """ - payload = { - "group": "scans", - "action": "get_scan_identified_components", - "data": { - "username": self.api_user, - "key": self.api_token, - "scan_code": scan_code, - }, - } - response = self._send_request(payload) - if response["status"] == "1" and "data" in response.keys(): - return response["data"] - raise builtins.Exception( - "Error getting identified components \ - result: {}".format( - response - ) - ) - - def get_scan_identified_licenses(self, scan_code: str): - """ - Retrieve the list of identified licenses from one scan. - - Args: - scan_code (str): The unique identifier for the scan. - - Returns: - dict: The JSON response from the API. - """ - payload = { - "group": "scans", - "action": "get_scan_identified_licenses", - "data": { - "username": self.api_user, - "key": self.api_token, - "scan_code": scan_code, - "unique": "1", - }, - } - response = self._send_request(payload) - if response["status"] == "1" and "data" in response.keys(): - return response["data"] - raise builtins.Exception( - "Error getting identified licenses \ - result: {}".format( - response - ) - ) - - def get_results(self, scan_code: str): - """ - Retrieve the list matches from one scan. - - Args: - scan_code (str): The unique identifier for the scan. - - Returns: - dict: The JSON response from the API. - """ - payload = { - "group": "scans", - "action": "get_results", - "data": { - "username": self.api_user, - "key": self.api_token, - "scan_code": scan_code, - "unique": "1", - }, - } - response = self._send_request(payload) - if response["status"] == "1" and "data" in response.keys(): - return response["data"] - raise builtins.Exception( - "Error getting scans ->get_results \ - result: {}".format( - response - ) - ) - - def _get_dependency_analysis_result(self, scan_code: str): - """ - Retrieve dependency analysis results. - - Args: - scan_code (str): The unique identifier for the scan. - - Returns: - dict: The JSON response from the API. - """ - payload = { - "group": "scans", - "action": "get_dependency_analysis_results", - "data": { - "username": self.api_user, - "key": self.api_token, - "scan_code": scan_code, - }, - } - response = self._send_request(payload) - if response["status"] == "1" and "data" in response.keys(): - return response["data"] - - raise builtins.Exception( - "Error getting dependency analysis \ - result: {}".format( - response - ) - ) - - def _cancel_scan(self, scan_code: str): - """ - Cancel a scan. - - Args: - scan_code (str): The unique identifier for the scan. - """ - payload = { - "group": "scans", - "action": "cancel_run", - "data": { - "username": self.api_user, - "key": self.api_token, - "scan_code": scan_code, - }, - } - response = self._send_request(payload) - if response["status"] != "1": - raise builtins.Exception("Error cancelling scan: {}".format(response)) - - def _assert_scan_can_start(self, scan_code: str): - """ - Verify if a new scan can be initiated. - - Args: - scan_code (str): The unique identifier for the scan. - """ - scan_status = self._get_scan_status("SCAN", scan_code) - # List of possible scan statuses taken from Workbench code: - # public const NEW = 'NEW'; - # public const QUEUED = 'QUEUED'; - # public const STARTING = 'STARTING'; - # public const RUNNING = 'RUNNING'; - # public const FINISHED = 'FINISHED'; - # public const FAILED = 'FAILED'; - if scan_status["status"] not in ["NEW", "FINISHED", "FAILED"]: - raise builtins.Exception( - "Cannot start scan. Current status of the scan is {}.".format( - scan_status["status"] - ) - ) - - def assert_dependency_analysis_can_start(self, scan_code: str): - """ - Verify if a new dependency analysis scan can be initiated. - - Args: - scan_code (str): The unique identifier for the scan. - """ - scan_status = self._get_scan_status("DEPENDENCY_ANALYSIS", scan_code) - # List of possible scan statuses taken from Workbench code: - # public const NEW = 'NEW'; - # public const QUEUED = 'QUEUED'; - # public const STARTING = 'STARTING'; - # public const RUNNING = 'RUNNING'; - # public const FINISHED = 'FINISHED'; - # public const FAILED = 'FAILED'; - if scan_status["status"] not in ["NEW", "FINISHED", "FAILED"]: - raise builtins.Exception( - "Cannot start dependency analysis. Current status of the scan is {}.".format( - scan_status["status"] - ) - ) - - def extract_archives( - self, - scan_code: str, - recursively_extract_archives: bool, - jar_file_extraction: bool, - ): - """ - Extract archive - - Args: - scan_code (str): The unique identifier for the scan. - recursively_extract_archives (bool): Yes or no - jar_file_extraction (bool): Yes or no - - Returns: - bool: true for successful API call - """ - payload = { - "group": "scans", - "action": "extract_archives", - "data": { - "username": self.api_user, - "key": self.api_token, - "scan_code": scan_code, - "recursively_extract_archives": recursively_extract_archives, - "jar_file_extraction": jar_file_extraction, - }, - } - response = self._send_request(payload) - if response["status"] == "0": - raise builtins.Exception( - "Call extract_archives returned error: {}".format(response) - ) - return True - - def check_if_scan_exists(self, scan_code: str): - """ - Check if scan exists. - - Args: - scan_code (str): The unique identifier for the scan. - - Returns: - bool: Yes or no. - """ - payload = { - "group": "scans", - "action": "get_information", - "data": { - "username": self.api_user, - "key": self.api_token, - "scan_code": scan_code, - }, - } - response = self._send_request(payload) - if response["status"] == "1": - return True - else: - return False - - def check_if_project_exists(self, project_code: str): - """ - Check if project exists. - - Args: - project_code (str): The unique identifier for the scan. - - Returns: - bool: Yes or no. - """ - payload = { - "group": "projects", - "action": "get_information", - "data": { - "username": self.api_user, - "key": self.api_token, - "project_code": project_code, - }, - } - response = self._send_request(payload) - if response["status"] == "0": - return False - # if response["status"] == "0": - # raise builtins.Exception("Failed to get project status: {}".format(response)) - return True - - def create_project(self, project_code: str): - """ - Create new project - - Args: - project_code (str): The unique identifier for the scan. - """ - payload = { - "group": "projects", - "action": "create", - "data": { - "username": self.api_user, - "key": self.api_token, - "project_code": project_code, - "project_name": project_code, - "description": "Automatically created by Workbench Agent script", - }, - } - response = self._send_request(payload) - if response["status"] != "1": - raise builtins.Exception("Failed to create project: {}".format(response)) - print("Created project {}".format(project_code)) - - def run_scan( - self, - scan_code: str, - limit: int, - sensitivity: int, - auto_identification_detect_declaration: bool, - auto_identification_detect_copyright: bool, - auto_identification_resolve_pending_ids: bool, - delta_only: bool, - reuse_identification: bool, - identification_reuse_type: str = None, - specific_code: str = None, - advanced_match_scoring: bool = True, - match_filtering_threshold: int = -1 - ): - """ - - Args: - scan_code (str): Unique scan identifier - limit (int): Limit the number of matches against the KB - sensitivity (int): Result sensitivity - auto_identification_detect_declaration (bool): Automatically detect license declaration inside files - auto_identification_detect_copyright (bool): Automatically detect copyright statements inside files - auto_identification_resolve_pending_ids (bool): Automatically resolve pending identifications - delta_only (bool): Scan only new or modified files - reuse_identification (bool): Reuse previous identifications - identification_reuse_type (str): Possible values: any,only_me,specific_project,specific_scan - specific_code (str): Fill only when reuse type: specific_project or specific_scan - advanced_match_scoring (bool): If true, scan will run with advanced match scoring. - match_filtering_threshold (int): Minimum length (in characters) of snippet to be considered - valid after applying intelligent match filtering. - Returns: - - """ - scan_exists = self.check_if_scan_exists(scan_code) - if not scan_exists: - raise builtins.Exception( - "Scan with scan_code: {} doesn't exist when calling 'run' action!".format( - scan_code - ) - ) - - self._assert_scan_can_start(scan_code) - print("Starting scan {}".format(scan_code)) - payload = { - "group": "scans", - "action": "run", - "data": { - "username": self.api_user, - "key": self.api_token, - "scan_code": scan_code, - "limit": limit, - "sensitivity": sensitivity, - "auto_identification_detect_declaration": int( - auto_identification_detect_declaration - ), - "auto_identification_detect_copyright": int( - auto_identification_detect_copyright - ), - "auto_identification_resolve_pending_ids": int( - auto_identification_resolve_pending_ids - ), - "delta_only": int(delta_only), - "advanced_match_scoring": int(advanced_match_scoring), - }, - } - if match_filtering_threshold > -1: - payload["data"]['match_filtering_threshold'] = match_filtering_threshold - if reuse_identification: - data = payload["data"] - data["reuse_identification"] = "1" - # 'any', 'only_me', 'specific_project', 'specific_scan' - if identification_reuse_type in {"specific_project", "specific_scan"}: - data["identification_reuse_type"] = identification_reuse_type - data["specific_code"] = specific_code - else: - data["identification_reuse_type"] = identification_reuse_type - - response = self._send_request(payload) - if response["status"] != "1": - logger.error( - "Failed to start scan {}: {} payload {}".format( - scan_code, response, payload - ) - ) - raise builtins.Exception( - "Failed to start scan {}: {}".format(scan_code, response["error"]) - ) - return response - - def remove_uploaded_content(self, filename: str, scan_code: str): - """ - When using chunked uploading every new chunk is appended to existing file, for this reason we need to make sure - that initially there is no file (from previous uploading). - - Args: - filename (str): The file to be deleted - scan_code (str): The unique identifier for the scan. - """ - print("Called scans->remove_uploaded_content on file {}".format(filename)) - payload = { - "group": "scans", - "action": "remove_uploaded_content", - "data": { - "username": self.api_user, - "key": self.api_token, - "scan_code": scan_code, - "filename": filename, - }, - } - resp = self._send_request(payload) - if resp["status"] != "1": - print( - f"Cannot delete file {filename}, maybe is the first time when uploading this file? API response {resp}." - ) - - -class CliWrapper: - """ - A class to interact with the FossID CLI. - - Attributes: - cli_path (string): Path to the executable file "fossid" - config_path (string): Path to the configuration file "fossid.conf" - timeout (int): timeout for CLI expressed in seconds - """ - - # __parameters (dictionary): Dictionary of parameters passed to 'fossid-cli' - __parameters = {} - - def __init__(self, cli_path, config_path, timeout="120"): - self.cli_path = cli_path - self.config_path = config_path - self.timeout = timeout - - # Executes fossid-cli --version - # Returns string - def get_version(self): - """ - Get CLI version - - Args: - self - - Returns: - str - """ - args = ["timeout", self.timeout, self.cli_path, "--version"] - try: - result = subprocess.check_output(args, stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as e: - return ( - "Calledprocerr: " - + str(e.cmd) - + " " - + str(e.returncode) - + " " - + str(e.output) - ) - # pylint: disable-next=broad-except - except Exception as e: - return "Error: " + str(e) - - return result - - def blind_scan(self, path, run_dependency_analysis): - """ - Call fossid-cli on a given path in order to generate hashes of the files from that path - - Args: - run_dependency_analysis (bool): whether to run dependency analysis or not - path (str): path of the code to be scanned - - Returns: - str: path to temporary .fossid file containing generated hashes - """ - temporary_file_path = "/tmp/blind_scan_result_" + self.randstring(8) + ".fossid" - # Create temporary file, make it empty if already exists - # pylint: disable-next=consider-using-with,unspecified-encoding - open(temporary_file_path, "w").close() - my_cmd = f"timeout {self.timeout} {self.cli_path} --local --enable-sha1=1 " - - if run_dependency_analysis: - my_cmd += " --dependency-analysis=1 " - - my_cmd += f" {path} > {temporary_file_path}" - - try: - # pylint: disable-next=unspecified-encoding - with open(temporary_file_path, "w") as outfile: - subprocess.check_output(my_cmd, shell=True, stderr=outfile) - # result = subprocess.check_output(args, stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as e: - print( - "Calledprocerr: " - + str(e.cmd) - + " " - + str(e.returncode) - + " " - + str(e.output) - ) - print(traceback.format_exc()) - sys.exit() - # pylint: disable-next=broad-except - except Exception as e: - print("Error: " + str(e)) - print(traceback.format_exc()) - sys.exit() - - return temporary_file_path - - @staticmethod - def randstring(length=10): - """ - Generate a random string of a given length - - Parameters: - length (int): Length of the generated string - - Returns: - str - """ - valid_letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - return "".join((random.choice(valid_letters) for i in range(0, length))) - - -def parse_cmdline_args(): - """ - Parses command line arguments for the script. - - Returns: - argparse.Namespace: An object containing the parsed command line arguments. - """ - - # Define a custom type function which will verify for empty string - def non_empty_string(s): - if not s.strip(): - raise argparse.ArgumentTypeError("Argument cannot be empty or just whitespace.") - return s - - parser = argparse.ArgumentParser( - add_help=False, - description="Run FossID Workbench Agent", - formatter_class=RawTextHelpFormatter, - ) - required = parser.add_argument_group("required arguments") - optional = parser.add_argument_group("optional arguments") - - # Add back help - optional.add_argument( - "-h", - "--help", - action="help", - default=argparse.SUPPRESS, - help="show this help message and exit", - ) - - required.add_argument( - "--api_url", - help="URL of the Workbench API instance, Ex: https://myserver.com/api.php", - type=non_empty_string, - required=True, - ) - required.add_argument( - "--api_user", - help="Workbench user that will make API calls", - type=non_empty_string, - required=True, - ) - required.add_argument( - "--api_token", - help="Workbench user API token (Not the same with user password!!!)", - type=non_empty_string, - required=True, - ) - required.add_argument( - "--project_code", - help="Name of the project inside Workbench where the scan will be created.\n" - "If the project doesn't exist, it will be created", - type=non_empty_string, - required=True, - ) - required.add_argument( - "--scan_code", - help="The scan code user when creating the scan in Workbench. It can be based on some env var,\n" - "for example: ${BUILD_NUMBER}", - type=non_empty_string, - required=True, - ) - optional.add_argument( - "--limit", - help="Limits CLI results to N most significant matches (default: 10)", - type=int, - default=10, - ) - optional.add_argument( - "--sensitivity", - help="Sets snippet sensitivity to a minimum of N lines (default: 10)", - type=int, - default=10, - ) - optional.add_argument( - "--recursively_extract_archives", - help="Recursively extract nested archives. Default false.", - action="store_true", - default=False, - ) - optional.add_argument( - "--jar_file_extraction", - help="Control default behavior related to extracting jar files. Default false.", - action="store_true", - default=False, - ) - optional.add_argument( - "--blind_scan", - help="Call CLI and generate file hashes. Upload hashes and initiate blind scan.", - action="store_true", - default=False, - ) - - optional.add_argument( - "--run_dependency_analysis", - help="Initiate dependency analysis after finishing scanning for matches in KB.", - action="store_true", - default=False, - ) - optional.add_argument( - "--run_only_dependency_analysis", - help="Scan only for dependencies, no results from KB.", - action="store_true", - default=False, - ) - optional.add_argument( - "--auto_identification_detect_declaration", - help="Automatically detect license declaration inside files. This argument expects no value, not passing\n" - "this argument is equivalent to assigning false.", - action="store_true", - default=False, - ) - optional.add_argument( - "--auto_identification_detect_copyright", - help="Automatically detect copyright statements inside files. This argument expects no value, not passing\n" - "this argument is equivalent to assigning false.", - action="store_true", - default=False, - ) - optional.add_argument( - "--auto_identification_resolve_pending_ids", - help="Automatically resolve pending identifications. This argument expects no value, not passing\n" - "this argument is equivalent to assigning false.", - action="store_true", - default=False, - ) - optional.add_argument( - "--delta_only", - help="""Scan only delta (newly added files from last scan).""", - action="store_true", - default=False, - ) - optional.add_argument( - "--reuse_identifications", - help="If present, try to use an existing identification depending on parameter ‘identification_reuse_type‘.", - action="store_true", - default=False, - required=False, - ) - optional.add_argument( - "--identification_reuse_type", - help="Based on reuse type last identification found will be used for files with the same hash.", - choices=["any", "only_me", "specific_project", "specific_scan"], - default="any", - type=str, - required=False, - ) - optional.add_argument( - "--specific_code", - help="The scan code used when creating the scan in Workbench. It can be based on some env var,\n" - "for example: ${BUILD_NUMBER}", - type=str, - required=False, - ) - optional.add_argument( - '--no_advanced_match_scoring', - help='Disable advanced match scoring which by default is enabled.', - dest='advanced_match_scoring', - action='store_false', - ) - optional.add_argument( - "--match_filtering_threshold", - help="Minimum length, in characters, of the snippet to be considered valid after applying match filtering.\n" - "Set to 0 to disable intelligent match filtering for current scan.", - type=int, - default=-1, - ) - optional.add_argument( - "--target_path", - help="The path on the Workbench server where the code to be scanned is stored.\n" - "No upload is done in this scenario.", - type=str, - required=False, - ) - optional.add_argument( - "--chunked_upload", - help="For files bigger than 8 MB (which is default post_max_size in php.ini) uploading will be done using\n" - "the header Transfer-encoding: chunked with chunks of 5MB.", - action="store_true", - default=False, - required=False, - ) - required.add_argument( - "--scan_number_of_tries", - help="""Number of calls to 'check_status' till declaring the scan failed from the point of view of the agent""", - type=int, - default=960, # This means 8 hours when --scan_wait_time has default value 30 seconds - required=False, - ) - required.add_argument( - "--scan_wait_time", - help="Time interval between calling 'check_status', expressed in seconds (default 30 seconds)", - type=int, - default=30, - required=False, - ) - required.add_argument( - "--path", - help="Path of the directory where the files to be scanned reside", - type=str, - required=True, - ) - - optional.add_argument( - "--log", - help="specify logging level. Allowed values: DEBUG, INFO, WARNING, ERROR", - default="ERROR", - ) - - optional.add_argument( - "--path-result", - help="Save results to specified path", - type=str, - required=False, - ) - - optional.add_argument( - "--get_scan_identified_components", - help="By default at the end of scanning the list of licenses identified will be retrieved.\n" - "When passing this parameter the agent will return the list of identified components instead.\n" - "This argument expects no value, not passing this argument is equivalent to assigning false.", - action="store_true", - default=False, - ) - optional.add_argument( - "--scans_get_policy_warnings_counter", - help="By default at the end of scanning the list of licenses identified will be retrieved.\n" - "When passing this parameter the agent will return information about policy warnings found in this scan\n" - "based on policy rules set at Project level.\n" - "This argument expects no value, not passing this argument is equivalent to assigning false.", - action="store_true", - default=False, - ) - optional.add_argument( - "--projects_get_policy_warnings_info", - help="By default at the end of scanning the list of licenses identified will be retrieved.\n" - "When passing this parameter the agent will return information about policy warnings for project,\n" - "including the warnings counter.\n" - "This argument expects no value, not passing this argument is equivalent to assigning false.", - action="store_true", - default=False, - ) - optional.add_argument( - "--scans_get_results", - help="By default at the end of scanning the list of licenses identified will be retrieved.\n" - "When passing this parameter the agent will return information about policy warnings found in this scan\n" - "based on policy rules set at Project level.\n" - "This argument expects no value, not passing this argument is equivalent to assigning false.", - action="store_true", - default=False, - ) - - args = parser.parse_args() - return args - - -def save_results(params, results): - """ - Saves the scanning results to a specified path. - - Parameters: - params (argparse.Namespace): Parsed command line parameters. - results (dict): The scan results to be saved. - """ - if params.path_result: - if os.path.isdir(params.path_result): - fname = os.path.join(params.path_result, "wb_results.json") - try: - with open(fname, "w") as file: - file.write(json.dumps(results, indent=4)) - print(f"Results saved to: {fname}") - except builtins.Exception: - logger.debug(f"Error trying to write results to {fname}") - print(f"Error trying to write results to {fname}") - elif os.path.isfile(params.path_result): - fname = params.path_result - _folder = os.path.dirname(params.path_result) - _fname = os.path.basename(params.path_result) - if _fname: - if not _fname.endswith(".json"): - try: - extension = _fname.split(".")[-1] - _fname = _fname.replace(extension, "json") - except (TypeError, IndexError): - _fname = f"{_fname.replace('.', '_')}.json" - else: - _fname = "wb_results.json" - try: - os.makedirs(_folder, exist_ok=True) - try: - with open(fname, "w") as file: - file.write(json.dumps(results, indent=4)) - print(f"Results saved to: {fname}") - except builtins.Exception: - logger.debug(f"Error trying to write results to {fname}") - except PermissionError: - logger.debug(f"Error trying to create folder: {_folder}") - else: - logger.debug(f"Folder or file does not exist: {params.path_result}") - try: - fname = params.path_result - if fname.endswith(".json"): - _folder = os.path.dirname(fname) - else: - if "." in fname: - _folder = os.path.dirname(fname) - else: - _folder = fname - fname = os.path.join(_folder, "wb_results.json") - try: - os.makedirs(_folder, exist_ok=True) - try: - with open(fname, "w") as file: - file.write(json.dumps(results, indent=4)) - print(f"Results saved to: {fname}") - except builtins.Exception: - logger.debug(f"Error trying to write results to {fname}") - except builtins.Exception: - logger.debug(f"Error trying to create folder: {_folder}") - except builtins.Exception: - logger.debug(f"Error trying to create report: {params.path_result}") - - -def main(): - # Retrieve parameters from command line - params = parse_cmdline_args() - logger.setLevel(params.log) - f_handler = logging.FileHandler("log-agent.txt") - logger.addHandler(f_handler) - - # Display parsed parameters - print("Parsed parameters: ") - for k, v in params.__dict__.items(): - print("{} = {}".format(k, v)) - - if params.blind_scan: - cli_wrapper = CliWrapper("/usr/bin/fossid-cli", "/etc/fossid.conf") - # Display fossid-cli version just to validate the path to CLI - print(cli_wrapper.get_version()) - - # Run scan and save .fossid file as temporary file - blind_scan_result_path = cli_wrapper.blind_scan(params.path, params.run_dependency_analysis) - print( - "Temporary file containing hashes generated at path: {}".format( - blind_scan_result_path - ) - ) - - # Create Project if it doesn't exist - workbench = Workbench(params.api_url, params.api_user, params.api_token) - if not workbench.check_if_project_exists(params.project_code): - workbench.create_project(params.project_code) - # Create scan if it doesn't exist - scan_exists = workbench.check_if_scan_exists(params.scan_code) - if not scan_exists: - print( - f"Scan with code {params.scan_code} does not exist. Calling API to create it..." - ) - workbench.create_webapp_scan(params.scan_code, params.project_code, params.target_path) - else: - print( - f"Scan with code {params.scan_code} already exists. Proceeding to upload..." - ) - # Handle blind scan differently from regular scan - if params.blind_scan: - # Upload temporary file with blind scan hashes - print("Parsed path: ", params.path) - workbench.upload_files(params.scan_code, blind_scan_result_path) - - # delete .fossid file containing hashes (after upload to scan) - if os.path.isfile(blind_scan_result_path): - os.remove(blind_scan_result_path) - else: - print( - "Can not delete the file {} as it doesn't exists".format( - blind_scan_result_path - ) - ) - # Handle normal scanning (directly uploading files at given path instead of generating hashes with CLI) - # There is no file upload when scanning from target path - elif not params.target_path: - if not os.path.isdir(params.path): - # The given path is an actual file path. Only this file will be uploaded - print( - "Uploading file indicated in --path parameter: {}".format(params.path) - ) - workbench.upload_files(params.scan_code, params.path, params.chunked_upload) - else: - # Get all files found at given path (including in subdirectories). Exclude directories - print( - "Uploading files found in directory indicated in --path parameter: {}".format( - params.path - ) - ) - counter_files = 0 - for root, directories, filenames in os.walk(params.path): - for filename in filenames: - if not os.path.isdir(os.path.join(root, filename)): - counter_files = counter_files + 1 - workbench.upload_files( - params.scan_code, os.path.join(root, filename), params.chunked_upload - ) - print("A total of {} files uploaded".format(counter_files)) - print("Calling API scans->extracting_archives") - workbench.extract_archives( - params.scan_code, - params.recursively_extract_archives, - params.jar_file_extraction, - ) - - # If --run_only_dependency_analysis parameter is true ONLY run dependency analysis, no KB scanning - if params.run_only_dependency_analysis: - workbench.assert_dependency_analysis_can_start(params.scan_code) - print("Starting dependency analysis for scan: {}".format(params.scan_code)) - workbench.start_dependency_analysis(params.scan_code) - # Check if finished based on: scan_number_of_tries X scan_wait_time until throwing an error - workbench.wait_for_scan_to_finish( - "DEPENDENCY_ANALYSIS", - params.scan_code, - params.scan_number_of_tries, - params.scan_wait_time, - ) - # Run scan - else: - workbench.run_scan( - params.scan_code, - params.limit, - params.sensitivity, - params.auto_identification_detect_declaration, - params.auto_identification_detect_copyright, - params.auto_identification_resolve_pending_ids, - params.delta_only, - params.reuse_identifications, - params.identification_reuse_type, - params.specific_code, - params.advanced_match_scoring, - params.match_filtering_threshold - ) - # Check if finished based on: scan_number_of_tries X scan_wait_time until throwing an error - workbench.wait_for_scan_to_finish( - "SCAN", params.scan_code, params.scan_number_of_tries, params.scan_wait_time - ) - - # If --run_dependency_analysis parameter is true run also dependency analysis - if params.run_dependency_analysis: - workbench.assert_dependency_analysis_can_start(params.scan_code) - print("Starting dependency analysis for scan: {}".format(params.scan_code)) - workbench.start_dependency_analysis(params.scan_code) - # Check if finished based on: scan_number_of_tries X scan_wait_time until throwing an error - workbench.wait_for_scan_to_finish( - "DEPENDENCY_ANALYSIS", - params.scan_code, - params.scan_number_of_tries, - params.scan_wait_time, - ) - - # When scan finished retrieve licenses list by default of if parameter --get_scan_identified_components is True call - # scans -> get_scan_identified_components - if params.get_scan_identified_components: - print("Identified components: ") - identified_components = workbench.get_scan_identified_components( - params.scan_code - ) - print(json.dumps(identified_components)) - save_results(params=params, results=identified_components) - sys.exit(0) - - # projects -> get_policy_warnings_info - elif params.scans_get_policy_warnings_counter: - if params.project_code is None or params.project_code == "": - print( - "Parameter project_code missing!\n" - "In order for the scans->get_policy_warnings_counter to be called a project code is required." - ) - sys.exit(1) - print(f"Scan: {params.scan_code} policy warnings info: ") - info_policy = workbench.scans_get_policy_warnings_counter(params.scan_code) - print(json.dumps(info_policy)) - save_results(params=params, results=info_policy) - sys.exit(0) - # When scan finished retrieve project policy warnings info - # projects -> get_policy_warnings_info - elif params.projects_get_policy_warnings_info: - if params.project_code is None or params.project_code == "": - print( - "Parameter project_code missing!\n" - "In order for the projects->get_policy_warnings_info to be called a project code is required." - ) - sys.exit(1) - print(f"Project {params.project_code} policy warnings info: ") - info_policy = workbench.projects_get_policy_warnings_info(params.project_code) - print(json.dumps(info_policy)) - save_results(params=params, results=info_policy) - sys.exit(0) - # When scan finished retrieve project policy warnings info - # projects -> get_policy_warnings_info - elif params.scans_get_results: - - print(f"Scan {params.scan_code} results: ") - results = workbench.get_results(params.scan_code) - print(json.dumps(results)) - save_results(params=params, results=results) - sys.exit(0) - else: - print("Identified licenses: ") - identified_licenses = workbench.get_scan_identified_licenses(params.scan_code) - print(json.dumps(identified_licenses)) - save_results(params=params, results=identified_licenses) - - -main() +if __name__ == "__main__": + sys.exit(main()) # Call the main function and exit with its code