Release 3.0.2

.github/workflows/benchmark.yml

@@ -25,7 +25,7 @@ jobs:
         fetch-depth: 0  # Need full history for branch comparison
 
     - name: Set up Python
-      uses: actions/setup-python@v6
+      uses: actions/setup-python@v7
       with:
         python-version: "3.11"
 
@@ -42,10 +42,13 @@ jobs:
 
     # Generate benchmark comparison report using our branch-based script
     - name: Generate benchmark comparison report
+      env:
+        BASE_REF: ${{ github.base_ref }}
+        HEAD_REF: ${{ github.head_ref }}
       run: |
         uv run python bbot/scripts/benchmark_report.py \
-          --base ${{ github.base_ref }} \
-          --current ${{ github.head_ref }} \
+          --base "$BASE_REF" \
+          --current "$HEAD_REF" \
           --output benchmark_report.md \
           --keep-results
       continue-on-error: true

.github/workflows/docs_updater.yml

@@ -14,7 +14,7 @@ jobs:
           token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
           ref: dev  # Checkout the dev branch
       - name: Set up Python
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v7
         with:
           python-version: "3.x"
       - name: Install uv

.github/workflows/tests.yml

@@ -21,7 +21,7 @@ jobs:
     steps:
       - uses: actions/checkout@v7
       - name: Set up Python
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v7
         with:
           python-version: ${{ matrix.python-version }}
       - name: Set Python Version Environment Variable
@@ -68,7 +68,7 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Set up Python
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v7
         with:
           python-version: "3.x"
       - name: Install uv
@@ -228,7 +228,7 @@ jobs:
       - uses: actions/checkout@v7
         with:
           token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
-      - uses: actions/setup-python@v6
+      - uses: actions/setup-python@v7
         with:
           python-version: "3.11"
       - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV

.github/workflows/version_updater.yml

@@ -15,7 +15,7 @@ jobs:
           fetch-depth: 0
           token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
       - name: Set up Python
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v7
         with:
           python-version: '3.x'
       - name: Install dependencies
@@ -67,7 +67,7 @@ jobs:
           fetch-depth: 0
           token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
       - name: Set up Python
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v7
         with:
           python-version: '3.x'
       - name: Install dependencies

bbot/core/helpers/depsinstaller/installer.py

@@ -15,6 +15,8 @@
 from secrets import token_bytes
 from ansible_runner.interface import run
 from subprocess import CalledProcessError
+from importlib.metadata import PackageNotFoundError, version as installed_version
+from packaging.requirements import InvalidRequirement, Requirement
 
 from bbot import __version__
 from ..misc import can_sudo_without_password, os_platform, rm_at_exit, get_python_constraints
@@ -183,38 +185,43 @@ async def install(self, *modules):
                     log.debug(f'No dependency work to do for module "{m}"')
                     succeeded.append(m)
                     continue
-                else:
-                    if (
-                        success is None
-                        or (success is False and self.deps_behavior == "retry_failed")
-                        or self.deps_behavior == "force_install"
-                    ):
-                        if not notified:
-                            log.hugeinfo("Installing module dependencies. Please be patient, this may take a while.")
-                            notified = True
-                        log.verbose(f'Installing dependencies for module "{m}"')
-                        # get sudo access if we need it
-                        if preloaded.get("sudo", False) is True:
-                            self.ensure_root(f'Module "{m}" needs root privileges to install its dependencies.')
-                        success = await self.install_module(m)
-                        self.setup_status[module_hash] = success
-                        if success or self.deps_behavior == "ignore_failed":
-                            log.debug(f'Setup succeeded for module "{m}"')
-                            succeeded.append(m)
-                        else:
-                            log.error(f'Setup failed for module "{m}"')
-                            failed.append(m)
+                # don't trust the cache if the packages it claims to have installed are gone
+                # (e.g. the virtualenv was rebuilt by "uv sync")
+                if success is True:
+                    satisfied, reason = self._pip_deps_satisfied(preloaded["deps"]["pip"])
+                    if not satisfied:
+                        log.verbose(f'Dependencies for module "{m}" need reinstalling ({reason})')
+                        success = None
+                if (
+                    success is None
+                    or (success is False and self.deps_behavior == "retry_failed")
+                    or self.deps_behavior == "force_install"
+                ):
+                    if not notified:
+                        log.hugeinfo("Installing module dependencies. Please be patient, this may take a while.")
+                        notified = True
+                    log.verbose(f'Installing dependencies for module "{m}"')
+                    # get sudo access if we need it
+                    if preloaded.get("sudo", False) is True:
+                        self.ensure_root(f'Module "{m}" needs root privileges to install its dependencies.')
+                    success = await self.install_module(m)
+                    self.setup_status[module_hash] = success
+                    if success or self.deps_behavior == "ignore_failed":
+                        log.debug(f'Setup succeeded for module "{m}"')
+                        succeeded.append(m)
                     else:
-                        if success or self.deps_behavior == "ignore_failed":
-                            log.debug(
-                                f'Skipping dependency install for module "{m}" because it\'s already done (--force-deps to re-run)'
-                            )
-                            succeeded.append(m)
-                        else:
-                            log.error(
-                                f'Skipping dependency install for module "{m}" because it failed previously (--retry-deps to retry or --ignore-failed-deps to ignore)'
-                            )
-                            failed.append(m)
+                        log.error(f'Setup failed for module "{m}"')
+                        failed.append(m)
+                elif success or self.deps_behavior == "ignore_failed":
+                    log.debug(
+                        f'Skipping dependency install for module "{m}" because it\'s already done (--force-deps to re-run)'
+                    )
+                    succeeded.append(m)
+                else:
+                    log.error(
+                        f'Skipping dependency install for module "{m}" because it failed previously (--retry-deps to retry or --ignore-failed-deps to ignore)'
+                    )
+                    failed.append(m)
 
         finally:
             self.write_setup_status()
@@ -271,7 +278,7 @@ async def pip_install(self, packages, constraints=None):
         command = [sys.executable, "-m", "pip", "install", "--upgrade"] + packages
 
         # if no custom constraints are provided, use the constraints of the currently installed version of bbot
-        if constraints is not None:
+        if not constraints:
             constraints = get_python_constraints()
 
         constraints_tempfile = self.parent_helper.tempfile(constraints, pipe=False)
@@ -465,6 +472,29 @@ def _core_dep_satisfied(self, command):
             )
         return bool(self.parent_helper.which(command))
 
+    def _pip_deps_satisfied(self, deps_pip):
+        """Check whether a module's pip dependencies are currently installed in this environment.
+
+        Returns (success, reason). Unparseable requirements (e.g. VCS URLs) can't be checked,
+        so they're assumed satisfied.
+        """
+        for dep in deps_pip:
+            try:
+                requirement = Requirement(dep)
+            except InvalidRequirement:
+                log.debug(f'Unable to verify pip dependency "{dep}"; assuming it is installed')
+                continue
+            # skip deps that don't apply to this interpreter/platform
+            if requirement.marker is not None and not requirement.marker.evaluate():
+                continue
+            try:
+                version = installed_version(requirement.name)
+            except PackageNotFoundError:
+                return False, f'pip package "{requirement.name}" is not installed'
+            if not requirement.specifier.contains(version, prereleases=True):
+                return False, f'pip package "{requirement.name}=={version}" does not satisfy "{dep}"'
+        return True, ""
+
     async def install_core_deps(self):
         # skip if we've already successfully installed core deps for this definition
         core_deps_hash = str(mmh3.hash(orjson.dumps(self.CORE_DEPS, option=orjson.OPT_SORT_KEYS)))

bbot/modules/docker_pull.py

@@ -81,7 +81,10 @@ async def docker_api_request(self, url: str):
         """Make a request to the URL if that fails try to obtain an authentication token and try again."""
         for _ in range(2):
             response = await self.helpers.request(url, headers=self.headers, follow_redirects=True)
-            if response is not None and response.status_code != 401:
+            if response is None:
+                self.log.warning(f"Request to {url} failed")
+                break
+            if response.status_code != 401:
                 return response
             www_auth = response.headers.get("www-authenticate", "")
             realm, service, scope = self._parse_www_authenticate(www_auth)

bbot/modules/trufflehog.py

@@ -15,7 +15,7 @@ class trufflehog(BaseModule):
     }
 
     class Config(BaseModuleConfig):
-        version: str = Field("3.95.9", description="trufflehog version")
+        version: str = Field("3.96.0", description="trufflehog version")
         config: str = Field("", description="File path or URL to YAML trufflehog config")
         only_verified: bool = Field(True, description="Only report credentials that have been verified")
         concurrency: int = Field(8, description="Number of concurrent workers")
@@ -37,6 +37,7 @@ class Config(BaseModuleConfig):
     ]
 
     scope_distance_modifier = 2
+    _module_threads = 2
 
     async def setup_deps(self):
         self.config_file = self.config.get("config", "")
@@ -86,6 +87,8 @@ async def handle_event(self, event):
         if isinstance(event.data, dict):
             description = event.data.get("description", "")
 
+        path = None
+        stdin_data = None
         if event.type == "CODE_REPOSITORY":
             path = event.url
             module = "github-experimental"
@@ -100,12 +103,8 @@ async def handle_event(self, event):
             else:
                 module = "filesystem"
         elif event.type in ("HTTP_RESPONSE", "RAW_TEXT"):
-            module = "filesystem"
-            file_data = event.raw_response if event.type == "HTTP_RESPONSE" else event.data
-            # write the response to a tempfile
-            # this is necessary because trufflehog doesn't yet support reading from stdin
-            # https://github.com/trufflesecurity/trufflehog/issues/162
-            path = self.helpers.tempfile(file_data, pipe=False)
+            module = "stdin"
+            stdin_data = event.raw_response if event.type == "HTTP_RESPONSE" else event.data
 
         if event.type == "CODE_REPOSITORY":
             host = event.host
@@ -118,7 +117,7 @@ async def handle_event(self, event):
             rawv2_result,
             verified,
             source_metadata,
-        ) in self.execute_trufflehog(module, path):
+        ) in self.execute_trufflehog(module, path=path, stdin_data=stdin_data):
             verified_str = "Verified" if verified else "Possible"
             confidence = "CONFIRMED" if verified else "MEDIUM"
             data = {
@@ -127,6 +126,10 @@ async def handle_event(self, event):
             }
             if host:
                 data["host"] = host
+            if event.type == "HTTP_RESPONSE":
+                url = event.data.get("url", "")
+                if url:
+                    data["url"] = url
 
             data["severity"] = "HIGH"
             data["confidence"] = confidence
@@ -142,11 +145,7 @@ async def handle_event(self, event):
                 context=f'{{module}} searched {event.type} using "{module}" method and found {verified_str.lower()} secret ({{event.type}}): {raw_result}',
             )
 
-        # clean up the tempfile when we're done with it
-        if event.type in ("HTTP_RESPONSE", "RAW_TEXT"):
-            path.unlink(missing_ok=True)
-
-    async def execute_trufflehog(self, module, path=None, string=None):
+    async def execute_trufflehog(self, module, path=None, stdin_data=None):
         command = [
             "trufflehog",
             "--json",
@@ -169,17 +168,23 @@ async def execute_trufflehog(self, module, path=None, string=None):
         elif module == "filesystem":
             command.append("filesystem")
             command.append(path)
+        elif module == "stdin":
+            command.append("stdin")
         elif module == "github-experimental":
             command.append("github-experimental")
             command.append("--repo=" + path)
             command.append("--object-discovery")
             command.append("--delete-cached-data")
             command.append("--token=" + self.github_token)
 
-        stats_file = self.helpers.tempfile_tail(callback=partial(self.log_trufflehog_status, path))
+        run_kwargs = {}
+        if stdin_data is not None:
+            run_kwargs["input"] = stdin_data
+
+        stats_file = self.helpers.tempfile_tail(callback=partial(self.log_trufflehog_status, path or module))
         try:
             with open(stats_file, "w") as stats_fh:
-                async for line in self.helpers.run_live(command, stderr=stats_fh):
+                async for line in self.run_process_live(command, stderr=stats_fh, **run_kwargs):
                     try:
                         j = json.loads(line)
                     except json.decoder.JSONDecodeError:
@@ -202,7 +207,7 @@ async def execute_trufflehog(self, module, path=None, string=None):
         finally:
             stats_file.unlink(missing_ok=True)
 
-    def log_trufflehog_status(self, path, line):
+    def log_trufflehog_status(self, target, line):
         try:
             line = json.loads(line)
         except Exception:
@@ -211,5 +216,5 @@ def log_trufflehog_status(self, path, line):
         message = line.get("msg", "")
         ts = line.get("ts", "")
         status = f"Message: {message} | Timestamp: {ts}"
-        self.verbose(f"Current scan target: {path}")
+        self.verbose(f"Current scan target: {target}")
         self.verbose(status)

bbot/presets/web/js-audit.yml

@@ -0,0 +1,20 @@
+description: Hunt for leaked credentials and vulnerable libraries in client-side JavaScript
+
+
+modules:
+  - http
+  - trufflehog
+  - badsecrets
+  - retirejs
+  - robots
+  - wayback
+
+config:
+  modules:
+    trufflehog:
+      only_verified: false
+    robots:
+      include_sitemap: true
+    wayback:
+      urls: true
+      archive: true