fix: preserve Zscaler pagination results

.pre-commit-config.yaml

@@ -33,7 +33,7 @@ repos:
         args: [ "--fix", "--unsafe-fixes"] # Allow unsafe fixes (ruff pretty strict about what it can fix)
       - id: ruff-format
   - repo: https://github.com/djlint/djLint
-    rev: v1.40.7
+    rev: v1.42.1
     hooks:
       - id: djlint-reformat-django
       - id: djlint-django
@@ -61,7 +61,7 @@ repos:
         exclude: "README.md"
   # Central hooks
   - repo: https://github.com/phantomcyber/dev-cicd-tools
-    rev: v2.1.6
+    rev: v2.2.8
     hooks:
       - id: build-docs
         language: python

release_notes/unreleased.md

@@ -1 +1,9 @@
 **Unreleased**
+
+* Refresh development validation tooling. [PSAAS-32647]
+
+* Continue Zscaler directory pagination when a response returns a short page. [PSAAS-32647]
+
+* Normalize URL schemes before applying Zscaler URL list actions. [PSAAS-32634]
+
+* Bound denylist regular-expression filtering to keep Zscaler actions responsive. [PSAAS-33137]

zscaler_connector.py

@@ -18,6 +18,8 @@
 import ipaddress
 import json
 import re
+import subprocess
+import sys
 import time
 from urllib.parse import quote
 
@@ -31,6 +33,42 @@
 from zscaler_consts import *
 
 
+def _filter_denylist_by_query(query, blocklist):
+    """Evaluate a denylist query outside the action worker with a fixed timeout."""
+    worker_code = """
+import json
+import re
+import sys
+
+payload = json.load(sys.stdin)
+try:
+    pattern = re.compile(payload["query"])
+    print(json.dumps({"matches": [entry for entry in payload["blocklist"] if pattern.fullmatch(entry)]}))
+except re.error:
+    print(json.dumps({"error": "invalid regular expression"}))
+"""
+    try:
+        completed = subprocess.run(
+            [sys.executable, "-c", worker_code],
+            capture_output=True,
+            check=True,
+            input=json.dumps({"query": query, "blocklist": blocklist}),
+            text=True,
+            timeout=5,
+        )
+        response = json.loads(completed.stdout)
+    except subprocess.TimeoutExpired:
+        return None, "Regular expression query timed out after 5 seconds."
+    except (OSError, subprocess.CalledProcessError, ValueError):
+        return None, "Unable to evaluate the regular expression query."
+
+    if "error" in response:
+        return None, "Invalid regular expression query."
+    if not isinstance(response.get("matches"), list):
+        return None, "Unable to evaluate the regular expression query."
+    return response["matches"], None
+
+
 class RetVal(tuple):
     def __new__(cls, val1, val2):
         return tuple.__new__(RetVal, (val1, val2))
@@ -759,9 +797,9 @@ def _truncate_protocol(self, endpoints):
         :return: updated list of url
         """
         for i in range(len(endpoints)):
-            if endpoints[i].startswith("http://"):
+            if endpoints[i].lower().startswith("http://"):
                 endpoints[i] = endpoints[i][(len("http://")) :]
-            elif endpoints[i].startswith("https://"):
+            elif endpoints[i].lower().startswith("https://"):
                 endpoints[i] = endpoints[i][(len("https://")) :]
 
         return endpoints
@@ -803,7 +841,7 @@ def _handle_get_admin_users(self, param):
                 return action_result.get_status()
             for admin_user in get_admin_users:
                 admin_users.append(admin_user)
-            limit = limit - params["pageSize"]
+            limit = limit - len(get_admin_users)
             if limit <= 0 or len(get_admin_users) == 0:
                 break
             params["page"] += 1
@@ -844,7 +882,7 @@ def _handle_get_users(self, param):
                 return action_result.get_status()
             for user in get_users:
                 users.append(user)
-            limit = limit - params["pageSize"]
+            limit = limit - len(get_users)
             if limit <= 0 or len(get_users) == 0:
                 break
             params["page"] += 1
@@ -881,7 +919,7 @@ def _handle_get_groups(self, param):
                 return action_result.get_status()
             for group in get_groups:
                 groups.append(group)
-            limit = limit - params["pageSize"]
+            limit = limit - len(get_groups)
             if limit <= 0 or len(get_groups) == 0:
                 break
             params["page"] += 1
@@ -1021,15 +1059,21 @@ def _handle_get_denylist(self, param):
         summary = action_result.update_summary({})
         summary["message"] = "Denylist retrieved"
 
-        blocklist = response.get("blacklistUrls", [])
-        for blocked in blocklist:
+        blocklist = []
+        for blocked in response.get("blacklistUrls", []):
             is_ip = self._is_ip_address(blocked)
             if filter == "ip" and not is_ip:
                 continue
             if filter == "url" and is_ip:
                 continue
-            if query and not re.fullmatch(query, blocked):
-                continue
+            blocklist.append(blocked)
+
+        if query:
+            blocklist, error_message = _filter_denylist_by_query(query, blocklist)
+            if error_message:
+                return action_result.set_status(phantom.APP_ERROR, error_message)
+
+        for blocked in blocklist:
             action_result.add_data({"url": blocked})
 
         summary["total_denylist_items"] = action_result.get_data_size()
@@ -1287,7 +1331,7 @@ def _get_batched_groups(self, endpoint, params, action_result):
                     for key in extensions:
                         group[key] = extensions[key]
                 action_result.add_data(group)
-            limit = limit - params["pageSize"]
+            limit = limit - len(get_groups)
             if limit <= 0 or len(get_groups) == 0:
                 break
             params["page"] += 1