xorbitsai · sqhyz55 · Mar 18, 2026 · Mar 18, 2026 · May 11, 2026 · May 11, 2026
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -33,3 +33,27 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 - **LanceDB user_id migration hardening**
   Startup and migration logic now include cross-process file locking, legacy `-1` orphan marker remapping to reserved int64 sentinel values, zero-progress loop protection, and shared embeddings-table listing utilities to avoid API-compat drift.
+
+### Added
+
+- **Knowledge Base web ingestion: JS-rendered crawling (Playwright)**
+  Web ingestion now supports an optional **JS-rendered** mode for SPA/JavaScript-heavy websites (e.g. pages where the initial HTML contains mostly `<head>` metadata and the real content is rendered by JavaScript).
+
+  If you enable the **"Render JS (Playwright)"** option in the Knowledge Base web ingestion UI:
+
+  - **Docker (recommended)**: the official backend image installs Chromium during build (see `docker/Dockerfile.backend`). If you see an error about missing Playwright browsers, you are likely using an older image tag. Fix by updating/rebuilding the backend image:
+
+    ```bash
+    docker compose pull backend
+    docker compose up -d backend
+    ```
+
+  - **Local (non-Docker)**: install Playwright browsers once:
+
+    ```bash
+    cd xagent
+    source .venv/bin/activate
+    playwright install chromium
+    ```
+
+  If browsers are missing, the backend returns an error like **"Playwright browsers are not installed"** with the command to run.
diff --git a/frontend/src/components/kb/knowledge-base-creation-dialog.tsx b/frontend/src/components/kb/knowledge-base-creation-dialog.tsx
@@ -105,6 +105,7 @@ export function KnowledgeBaseCreationDialog({ open, onOpenChange, onSuccess }: K
     request_delay: 1.0,
     timeout: 30,
     respect_robots_txt: true,
+    render_js: false,
   })
 
   // Cloud connect state
@@ -298,6 +299,7 @@ export function KnowledgeBaseCreationDialog({ open, onOpenChange, onSuccess }: K
       request_delay: 1.0,
       timeout: 30,
       respect_robots_txt: true,
+      render_js: false,
     })
     setCurrentStep(1)
   }
@@ -425,6 +427,7 @@ export function KnowledgeBaseCreationDialog({ open, onOpenChange, onSuccess }: K
       formData.append("request_delay", webIngestionConfig.request_delay.toString())
       formData.append("timeout", webIngestionConfig.timeout.toString())
       formData.append("respect_robots_txt", webIngestionConfig.respect_robots_txt.toString())
+      formData.append("render_js", webIngestionConfig.render_js.toString())
 
       appendIngestionConfigToFormData(formData, ingestionConfig)
 
@@ -841,6 +844,16 @@ export function KnowledgeBaseCreationDialog({ open, onOpenChange, onSuccess }: K
                         />
                       </div>
                     </div>
+                    <div className="flex items-center gap-2">
+                      <input
+                        type="checkbox"
+                        id="create-render-js"
+                        checked={webIngestionConfig.render_js}
+                        onChange={(e) => setWebIngestionConfig(prev => ({ ...prev, render_js: e.target.checked }))}
+                        className="w-4 h-4"
+                      />
+                      <Label htmlFor="create-render-js" className="cursor-pointer">{t("kb.dialog.webImport.advanced.renderJs")}</Label>
+                    </div>
                   </div>
                 )}
               </div>

diff --git a/frontend/src/components/kb/knowledge-base-detail.tsx b/frontend/src/components/kb/knowledge-base-detail.tsx
@@ -176,6 +176,7 @@ export function KnowledgeBaseDetailContent({ collectionName }: { collectionName:
     request_delay: 1.0,
     timeout: 30,
     respect_robots_txt: true,
+    render_js: false,
   })
 
   // Embedding models state
@@ -482,6 +483,7 @@ export function KnowledgeBaseDetailContent({ collectionName }: { collectionName:
       formData.append("request_delay", webIngestionConfig.request_delay.toString())
       formData.append("timeout", webIngestionConfig.timeout.toString())
       formData.append("respect_robots_txt", webIngestionConfig.respect_robots_txt.toString())
+      formData.append("render_js", webIngestionConfig.render_js.toString())
 
       // Add ingestion configuration
       appendIngestionConfigToFormData(formData, ingestionConfig)
@@ -538,6 +540,7 @@ export function KnowledgeBaseDetailContent({ collectionName }: { collectionName:
         request_delay: 1.0,
         timeout: 30,
         respect_robots_txt: true,
+        render_js: false,
       })
 
     } catch (err) {
@@ -1296,6 +1299,23 @@ export function KnowledgeBaseDetailContent({ collectionName }: { collectionName:
                             />
                           </div>
                         </div>
+                        <div className="flex items-center gap-2">
+                          <input
+                            type="checkbox"
+                            id="dialog-render-js"
+                            checked={webIngestionConfig.render_js}
+                            onChange={(e) =>
+                              setWebIngestionConfig((prev) => ({
+                                ...prev,
+                                render_js: e.target.checked,
+                              }))
+                            }
+                            className="w-4 h-4"
+                          />
+                          <Label htmlFor="dialog-render-js" className="cursor-pointer">
+                            {t("kb.dialog.webImport.advanced.renderJs")}
+                          </Label>
+                        </div>
                       </div>
                     </details>
 

diff --git a/frontend/src/i18n/locales/en.ts b/frontend/src/i18n/locales/en.ts
@@ -1709,6 +1709,7 @@ Build when you need.`
           timeoutSeconds: "Request Timeout (seconds)",
           sameDomainOnly: "Same Domain Only",
           respectRobotsTxt: "Respect robots.txt",
+          renderJs: "Enable JS rendering (Playwright)",
           hintMultiple: "Separate multiple patterns with commas",
           hintContentSelector: "CSS selector for main content",
         },

diff --git a/frontend/src/i18n/locales/zh.ts b/frontend/src/i18n/locales/zh.ts
@@ -1709,6 +1709,7 @@ Build when you need.`
           timeoutSeconds: "超时时间（秒）",
           sameDomainOnly: "仅同域名",
           respectRobotsTxt: "遵守 robots.txt",
+          renderJs: "启用 JS 渲染（Playwright）",
           hintMultiple: "多个模式用逗号分隔",
           hintContentSelector: "CSS 选择器，用于提取主内容",
         },

diff --git a/src/xagent/core/tools/core/RAG_tools/core/schemas.py b/src/xagent/core/tools/core/RAG_tools/core/schemas.py
@@ -1769,6 +1769,24 @@ class WebCrawlConfig(BaseModel):
         description="Whether to respect robots.txt rules",
     )
 
+    # JavaScript rendering (Playwright)
+    render_js: bool = Field(
+        default=False,
+        description=(
+            "Whether to render pages with a real browser (Playwright) to support SPA/JS-heavy sites. "
+            "When enabled, crawling fetches the post-rendered DOM instead of raw HTML."
+        ),
+    )
+    render_wait_until: Literal["load", "domcontentloaded", "networkidle"] = Field(
+        default="networkidle",
+        description="Playwright navigation wait strategy when render_js=True",
+    )
+    render_timeout_ms: int = Field(
+        default=30000,
+        ge=1,
+        description="Playwright navigation timeout in milliseconds when render_js=True",
+    )
+
 
 class CrawlResult(BaseModel):
     """Result of crawling a single page."""

diff --git a/src/xagent/core/tools/core/RAG_tools/pipelines/document_ingestion.py b/src/xagent/core/tools/core/RAG_tools/pipelines/document_ingestion.py
@@ -191,6 +191,7 @@ def run_document_ingestion(
     user_id: Optional[int] = None,
     is_admin: Optional[bool] = None,
     file_id: Optional[str] = None,
+    trace_id: Optional[str] = None,
 ) -> IngestionResult:
     """Public entrypoint for LangGraph-compatible ingestion tooling.
 
@@ -224,6 +225,7 @@ def run_document_ingestion(
         user_id=user_id,
         is_admin=is_admin,
         file_id=file_id,
+        trace_id=trace_id,
     )
 
 
@@ -445,10 +447,14 @@ def _handle_ingestion_error(
     vector_count: int,
     warnings: List[str],
     user_id: Optional[int] = None,
+    trace_id: Optional[str] = None,
 ) -> IngestionResult:
     """Unify error handling for the ingestion pipeline."""
     logger.exception(
-        "Document ingestion pipeline failed at step '%s': %s", current_step, exc
+        "Document ingestion pipeline failed at step '%s': %s",
+        current_step,
+        exc,
+        extra={"trace_id": trace_id, "collection": collection, "doc_id": doc_id},
     )
 
     status = "partial" if completed_steps else "error"
@@ -484,6 +490,7 @@ def process_document(
     user_id: Optional[int] = None,
     is_admin: bool = False,
     file_id: Optional[str] = None,
+    trace_id: Optional[str] = None,
 ) -> IngestionResult:
     """Execute the full ingestion pipeline for a document.
 
@@ -558,7 +565,11 @@ def process_document(
         # Step 0: Initialize/validate collection embedding configuration
         logger.info(
             "Step initialize_collection started",
-            extra={"collection": collection, "source_path": source_path},
+            extra={
+                "trace_id": trace_id,
+                "collection": collection,
+                "source_path": source_path,
+            },
         )
         init_start = time.time()
 
@@ -610,6 +621,7 @@ def process_document(
         logger.info(
             "Step initialize_collection completed",
             extra={
+                "trace_id": trace_id,
                 "collection": collection,
                 "embedding_model_id": selected_model_id,
                 "elapsed_ms": init_elapsed,
@@ -1292,4 +1304,5 @@ def _update_embedding_progress(current: int, total: int) -> None:
             vector_count=vector_count,
             warnings=warnings,
             user_id=user_id,
+            trace_id=trace_id,
         )