Merge branch 'develop' of https://github.com/bluewave-labs/verifywise into develop-saas

Author: Harsh

Clients/nginx.conf

@@ -17,7 +17,7 @@ server {
 
     # Proxy API requests to backend
     location /api {
-        proxy_pass http://localhost:3000;  # no trailing slash - keeps /api prefix
+        proxy_pass http://localhost:3000; # no trailing slash - keeps /api prefix
         proxy_http_version 1.1;
         proxy_set_header Host $host;
         proxy_set_header X-Real-IP $remote_addr;

EvalServer/Dockerfile

@@ -14,6 +14,9 @@ RUN pip install --no-cache-dir torch==2.5.1 --index-url https://download.pytorch
 # Copy the application source
 COPY ./EvalServer/src .
 
+# Copy the deepeval_engine module from EvaluationModule
+COPY ./EvaluationModule/src/deepeval_engine /app/deepeval_engine
+
 # Copy the datasets from EvaluationModule
 COPY ./EvaluationModule/data/datasets /app/datasets
 

EvalServer/src/database/migrations/versions/20251220_cleanup_public_schema.py

@@ -0,0 +1,46 @@
+"""Cleanup public schema - remove tables that should only exist in tenant schemas
+
+Revision ID: 20251220_cleanup_public
+Revises: 20251220_add_missing
+Create Date: 2025-12-20
+
+Purpose:
+- Remove fairness_runs from public (was never dropped from public, only from tenant schemas)
+- Remove deepeval_projects from public (should only exist in tenant schemas)
+- These tables were created by older migrations before multi-tenancy was properly implemented
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = "20251220_cleanup_public"
+down_revision: Union[str, None] = "20251220_missing_evals"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Remove tables from public schema that should only be in tenant schemas."""
+    # Drop fairness_runs from public - this was the old bias/fairness feature table
+    op.execute(sa.text('DROP TABLE IF EXISTS public.fairness_runs CASCADE'))
+    
+    # Drop deepeval_projects from public - should only exist in tenant schemas
+    op.execute(sa.text('DROP TABLE IF EXISTS public.deepeval_projects CASCADE'))
+    
+    # Also clean up any other legacy bias/fairness tables from public
+    op.execute(sa.text('DROP TABLE IF EXISTS public.bias_fairness_evaluations CASCADE'))
+    
+    # Drop indexes that may have been created in public
+    op.execute(sa.text('DROP INDEX IF EXISTS public.idx_deepeval_projects_tenant'))
+    op.execute(sa.text('DROP INDEX IF EXISTS public.idx_deepeval_projects_created_at'))
+
+
+def downgrade() -> None:
+    """Recreate tables in public if needed (not recommended)."""
+    # We don't want to recreate these in public - they belong in tenant schemas
+    # But for migration reversibility, we provide empty downgrade
+    pass
+

EvaluationModule/src/deepeval_engine/deepeval_evaluator.py

@@ -637,52 +637,52 @@ def evaluate_test_cases(
                     )
             else:
                 # Use standard single-turn metrics
-            for metric_name, metric in metrics_to_use:
-                try:
-                    # Some metrics require retrieval/context. If missing, skip gracefully.
-                    # RAG-specific metrics require context
-                    requires_context = metric_name in {"Faithfulness", "Context Relevancy", "Context Precision", "Context Recall"}
+                for metric_name, metric in metrics_to_use:
+                    try:
+                        # Some metrics require retrieval/context. If missing, skip gracefully.
+                        # RAG-specific metrics require context
+                        requires_context = metric_name in {"Faithfulness", "Context Relevancy", "Context Precision", "Context Recall"}
+                            
+                        retrieval_context = getattr(test_case, "retrieval_context", None)
+                        context = getattr(test_case, "context", None)
+                        has_context = bool(retrieval_context) or bool(context)
 
-                    retrieval_context = getattr(test_case, "retrieval_context", None)
-                    context = getattr(test_case, "context", None)
-                    has_context = bool(retrieval_context) or bool(context)
-                    
-                    if requires_context and not has_context:
-                        print(f"  Evaluating {metric_name}... ⏭ Skipped (no context)")
-                        metric_scores[metric_name] = {
-                            "score": None,
-                            "passed": False,
-                            "threshold": getattr(metric, "threshold", None),
-                            "skipped": True,
-                            "reason": "No retrieval/context provided",
-                        }
-                        continue
+                        if requires_context and not has_context:
+                            print(f"  Evaluating {metric_name}... ⏭ Skipped (no context)")
+                            metric_scores[metric_name] = {
+                                "score": None,
+                                "passed": False,
+                                "threshold": getattr(metric, "threshold", None),
+                                "skipped": True,
+                                "reason": "No retrieval/context provided",
+                            }
+                            continue
 
-                    print(f"  Evaluating {metric_name}...", end=" ")
+                        print(f"  Evaluating {metric_name}...", end=" ")
 
-                    metric.measure(test_case)
-                    score = metric.score
-                    passed = metric.is_successful()
+                        metric.measure(test_case)
+                        score = metric.score
+                        passed = metric.is_successful()
 
-                    metric_scores[metric_name] = {
-                        "score": round(score, 3) if score is not None else None,
-                        "passed": passed,
-                        "threshold": getattr(metric, "threshold", None),
-                        "reason": getattr(metric, 'reason', 'N/A')
-                    }
+                        metric_scores[metric_name] = {
+                            "score": round(score, 3) if score is not None else None,
+                            "passed": passed,
+                            "threshold": getattr(metric, "threshold", None),
+                            "reason": getattr(metric, 'reason', 'N/A')
+                        }
 
-                    status = "✓ PASS" if passed else "✗ FAIL"
-                    print(f"{status} (score: {score:.3f})")
+                        status = "✓ PASS" if passed else "✗ FAIL"
+                        print(f"{status} (score: {score:.3f})")
 
-                except Exception as e:
-                    error_msg = str(e)
-                    print(f"✗ Error: {error_msg}")
-                    metric_scores[metric_name] = {
-                        "score": None,
-                        "passed": False,
-                        "threshold": getattr(metric, "threshold", None),
-                        "error": str(e)
-                    }
+                    except Exception as e:
+                        error_msg = str(e)
+                        print(f"✗ Error: {error_msg}")
+                        metric_scores[metric_name] = {
+                            "score": None,
+                            "passed": False,
+                            "threshold": getattr(metric, "threshold", None),
+                            "error": str(e)
+                        }
 
             # Calculate basic statistics based on test case type
             if is_conversational and isinstance(test_case, ConversationalTestCase):

Servers/database/migrations/20251104000511-create-tables-for-file-manager.js

@@ -6,15 +6,6 @@ module.exports = {
   async up(queryInterface, Sequelize) {
     const transaction = await queryInterface.sequelize.transaction();
     try {
-      // Create ENUM type in public schema (shared across tenants)
-      await queryInterface.sequelize.query(
-        `DO $$ BEGIN
-          CREATE TYPE public.enum_file_manager_source AS ENUM ('file_manager', 'policy_editor');
-        EXCEPTION
-          WHEN duplicate_object THEN null;
-        END $$;`,
-        { transaction }
-      );
 
       const queries = [
         // Create file_manager table for organization-wide file storage
@@ -28,8 +19,7 @@ module.exports = {
           uploaded_by INTEGER NOT NULL REFERENCES public.users(id) ON DELETE CASCADE,
           upload_date TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
           org_id INTEGER NOT NULL REFERENCES public.organizations(id) ON DELETE CASCADE,
-          is_demo BOOLEAN NOT NULL DEFAULT FALSE,
-          source public.enum_file_manager_source DEFAULT 'file_manager'
+          is_demo BOOLEAN NOT NULL DEFAULT FALSE
         );`,
 
         // Create file_access_logs table for audit trail
@@ -92,12 +82,6 @@ module.exports = {
         }
       }
 
-      // Drop ENUM type
-      await queryInterface.sequelize.query(
-        `DROP TYPE IF EXISTS public.enum_file_manager_source;`,
-        { transaction }
-      );
-
       await transaction.commit();
     } catch (error) {
       await transaction.rollback();

Servers/database/migrations/20251222161420-create-enum_file_manager_source-for-file-source.js

@@ -0,0 +1,69 @@
+'use strict';
+const { getTenantHash } = require("../../dist/tools/getTenantHash");
+
+/** @type {import('sequelize-cli').Migration} */
+module.exports = {
+  async up(queryInterface, Sequelize) {
+    const transaction = await queryInterface.sequelize.transaction();
+    try {
+      // Create ENUM type in public schema (shared across tenants)
+      await queryInterface.sequelize.query(
+        `DO $$ BEGIN
+            CREATE TYPE public.enum_file_manager_source AS ENUM ('file_manager', 'policy_editor');
+          EXCEPTION
+            WHEN duplicate_object THEN null;
+          END $$;`,
+        { transaction }
+      );
+
+      const organizations = await queryInterface.sequelize.query(
+        `SELECT id FROM organizations;`,
+        { transaction }
+      );
+
+      for (let organization of organizations[0]) {
+        const tenantHash = getTenantHash(organization.id);
+        await queryInterface.sequelize.query(
+          `ALTER TABLE "${tenantHash}".file_manager
+            ADD COLUMN IF NOT EXISTS source public.enum_file_manager_source DEFAULT 'file_manager' NOT NULL;`,
+          { transaction }
+        );
+      }
+
+      await transaction.commit();
+    } catch {
+      await transaction.rollback();
+      throw error;
+    }
+  },
+
+  async down(queryInterface, Sequelize) {
+    const transaction = await queryInterface.sequelize.transaction();
+    try {
+      // Drop ENUM type
+      await queryInterface.sequelize.query(
+        `DROP TYPE IF EXISTS public.enum_file_manager_source;`,
+        { transaction }
+      );
+
+      const organizations = await queryInterface.sequelize.query(
+        `SELECT id FROM organizations;`,
+        { transaction }
+      );
+
+      for (let organization of organizations[0]) {
+        const tenantHash = getTenantHash(organization.id);
+        await queryInterface.sequelize.query(
+          `ALTER TABLE "${tenantHash}".file_manager
+            DROP COLUMN IF EXISTS source;`,
+          { transaction }
+        );
+      }
+
+      await transaction.commit();
+    } catch (error) {
+      await transaction.rollback();
+      throw error;
+    }
+  }
+};

docker-compose.prod.yml

@@ -14,10 +14,6 @@ services:
       - $FRONTEND_PORT:80
     env_file:
       - ./.env.prod
-
-  # bias_and_fairness_backend:
-  #   env_file:
-  #     - ./.env.prod
 
   eval_server:
     env_file: