Merge branch 'main' of github.com:Zipstack/visitran into fix/global-schema-state-with-info-chip

Author: tahierhussain

.github/actions/setup-python/action.yaml

@@ -1,5 +1,5 @@
 name: 'Setup Python'
-description: 'Prints a greeting message'
+description: 'Sets up Python, uv, and installs backend dependencies'
 inputs:
   python-version:
     required: true
@@ -16,7 +16,8 @@ runs:
       with:
         python-version: ${{ inputs.python-version }}
         enable-cache: true
-        cache-dependency-path: ./uv.lock
-    - name: Install visitran-cloud dependencies
+        cache-dependency-path: backend/uv.lock
+    - name: Install backend dependencies
       shell: bash
-      run: uv sync
+      working-directory: backend
+      run: uv sync --group test

.github/pull_request_template.md

@@ -16,11 +16,11 @@
 
 ## Database Migrations
 
-- 
+-
 
 ## Env Config
 
-- 
+-
 
 ## Relevant Docs
 
@@ -42,4 +42,4 @@
 
 ## Checklist
 
-I have read and understood the [Contribution Guidelines]().
+I have read and understood the [Contribution Guidelines]().

.github/workflows/core-backend-full-tests-parallel.yaml

@@ -1,73 +1,71 @@
 ---
+name: Minimal Tests
+
+on:
+  workflow_dispatch:
+  push:
+    branches: ["main"]
+  pull_request:
+    branches: ["main"]
+
+concurrency:
+  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+env:
+  FORCE_COLOR: "1"
+
+jobs:
+  minimal_tests:
     name: Minimal Tests
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+      matrix:
+        python-version: ["3.10"]
+    defaults:
+      run:
+        working-directory: backend
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          lfs: true
+
+      - name: Setup Python
+        uses: ./.github/actions/setup-python/
+        with:
+          python-version: ${{ matrix.python-version }}
 
-    on:
-      workflow_dispatch:
-      push:
-        branches: ["main"]
-      pull_request:
-        branches: [ "main"]
+      # pre-commit checks handled by pre-commit.ci
 
-    concurrency:
-      group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
-      cancel-in-progress: true
-    env:
-      FORCE_COLOR: "1"
-    jobs:
-      minimal_tests:
-        name: Minimal Tests
-        runs-on: ubuntu-latest
-        strategy:
-          fail-fast: true
-          matrix:
-            python-version: ["3.11"] #,"3.9","3.10"]
-        steps:
-          #----------------------------------------------
-          #       check-out repo and set-up python
-          #----------------------------------------------
-          - uses: actions/checkout@v4
-            with:
-              lfs: true
-          - name: Setup Python
-            uses: ./.github/actions/setup-python/
-            with:
-              python-version: ${{ matrix.python-version }}
-          - name: Cache pre-commit hooks
-            uses: actions/cache@v4
-            if: github.ref != 'refs/heads/main'
-            with:
-              path: ~/.cache/pre-commit
-              key: ${{ runner.os }}-pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}
+      - name: Run tests
+        env:
+          DJANGO_SETTINGS_MODULE: backend.server.settings.dev
+        run: |
+          uv run coverage run --rcfile=pyproject.toml --source=. -m \
+          pytest -x -vv ../tests/unit \
+          --ignore=../tests/unit/test_logs.py \
+          --ignore=../tests/unit/test_visitran_adapters \
+          --ignore=../tests/unit/test_visitran_backend \
+          -m "not snowflake and not bigquery and not trino and not postgres"
 
-          - name: Check pre-commit
-            if: github.ref != 'refs/heads/main'
-            run: |
-              uv run pre-commit run --all-files
-          - name: Run minimal core tests
-            run: |
-              uv run coverage run --rcfile=pyproject.toml --data-file=minimal.cov --context="minimal" --source=. -m \
-              pytest -x -vv -m "not snowflake and not bigquery and not trino and not postgres"
-          - name: Run backend tests
-            run: |
-              uv run coverage run --rcfile=pyproject.toml --data-file=backend.cov --context="backend" --source=. -m \
-              pytest -x -vv ./visitran_backend
+      - name: Generate coverage report
+        run: |
+          uv run coverage report -m
+          uv run coverage xml
 
-          - name: Combine to coverage xml
-            run: |
-              uv run coverage combine backend.cov minimal.cov
-              uv run coverage report -m
-              uv run coverage xml
-          - name: Git fetch unshallow
-            run: |
-              git fetch --unshallow
+      - name: Git fetch unshallow
+        working-directory: .
+        run: git fetch --unshallow
 
-          - name: Core SonarCloud Scan
-            uses: SonarSource/sonarcloud-github-action@master
-            if: ${{ github.actor != 'dependabot[bot]' }}
-            env:
-              GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any
-              SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
-            with:
-              projectBaseDir: ./
-              args: >
-                -Dproject.settings=./sonar-project.properties
+      - name: SonarCloud Scan
+        uses: SonarSource/sonarcloud-github-action@v3
+        if: ${{ github.actor != 'dependabot[bot]' }}
+        continue-on-error: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+        with:
+          projectBaseDir: ./
+          args: >
+            -Dproject.settings=./sonar-project.properties

.github/workflows/core-backend-tests.yaml

@@ -213,4 +213,4 @@ backend/backend/utils/load_models/yaml_models.yaml
 
 # macOS
 .DS_Store
-**/.DS_Store
+**/.DS_Store

.gitignore

@@ -7,7 +7,17 @@ ci:
     - mypy # Uses language: system, not available in pre-commit.ci sandbox
     - protolint-docker # Needs Docker, not available in pre-commit.ci
     - hadolint-docker # Needs Docker, not available in pre-commit.ci
-  autofix_prs: true
+    - pycln # Path resolution bug in pre-commit.ci sandbox
+    - end-of-file-fixer # Inconsistent behavior between local and CI environments
+    - flake8 # Pre-existing violations — will clean up separately
+    - markdownlint # Pre-existing violations — will clean up separately
+    - absolufy-imports # Incorrectly rewrites relative imports in monorepo structure
+    - black # Large-scale reformatting — will clean up in dedicated PR
+    - pyupgrade # Pre-existing across 43 files
+    - isort # Pre-existing import ordering across codebase
+    - yesqa # Pre-existing unnecessary noqa comments
+    - docformatter # Pre-existing docstring formatting
+  autofix_prs: false
   autoupdate_schedule: monthly
 
 # Force all unspecified python hooks to run python 3.10
@@ -22,7 +32,10 @@ repos:
       - id: trailing-whitespace
         exclude_types:
           - "markdown"
+          - "svg"
       - id: end-of-file-fixer
+        exclude_types:
+          - "svg"
       - id: check-yaml
         args: [--unsafe]
       - id: check-added-large-files
@@ -36,6 +49,7 @@ repos:
       - id: check-toml
       - id: debug-statements
       - id: detect-private-key
+        exclude: sample\.env$
       # - id: detect-aws-credentials
       #   args: ["--allow-missing-credentials"]
       - id: check-merge-conflict
@@ -54,7 +68,7 @@ repos:
     rev: 24.1.1
     hooks:
       - id: black
-        args: [--config=pyproject.toml]
+        args: [--config=backend/pyproject.toml]
   # - id: black
   #   alias: black-check
   #   stages: [manual]
@@ -82,13 +96,13 @@ repos:
     rev: v2.4.0
     hooks:
       - id: pycln
-        args: [--config=pyproject.toml]
+        args: [--config=backend/pyproject.toml]
   - repo: https://github.com/pycqa/isort
     rev: 5.13.2
     hooks:
       - id: isort
         files: "\\.(py)$"
-        args: [--settings-path=pyproject.toml]
+        args: [--settings-path=backend/pyproject.toml]
   - repo: https://github.com/pycqa/flake8
     rev: 7.0.0
     hooks:
@@ -161,7 +175,7 @@ repos:
     - id: markdownlint
       args: ["--config", "markdownlint.yaml"]
   - repo: https://github.com/pycqa/docformatter
-    rev: v1.7.5
+    rev: v1.7.7
     hooks:
       - id: docformatter
   - repo: https://github.com/adrienverge/yamllint

.pre-commit-config.yaml

@@ -91,7 +91,7 @@ def unique_keys(self) -> list[str]:
     @property
     def delta_strategy(self) -> dict[str, Any]:
         return self.incremental_config.get("delta_strategy", {})
-    
+
     @property
     def reference(self) -> list[str]:
         if not self._reference:

backend/backend/application/config_parser/config_parser.py

@@ -318,6 +318,15 @@
         "description": "Tests whether a number is greater than a supplied threshold value ",
         "function_type": "Math",
     },
+    "HASH": {
+        "key": "HASH",
+        "value": "HASH",
+        "description": (
+            "Returns a hash of the concatenated values. Accepts one or more columns."
+            " \n Example: HASH(col1) or HASH(col1, col2, col3)"
+        ),
+        "function_type": "Text",
+    },
     "HOUR": {
         "key": "HOUR",
         "value": "HOUR",

backend/backend/application/config_parser/constants.py

@@ -65,19 +65,19 @@ def transform_orders(self) -> list[str]:
     def get_transforms(self) -> list[BaseParser]:
         """
         Generate and yield transformation parsers in the order defined by the configuration.
-    
-        This method processes the `transform_order` list and corresponding `transform` dictionary 
+
+        This method processes the `transform_order` list and corresponding `transform` dictionary
         from the configuration to create parser instances of appropriate types for each transformation.
-    
+
         - It iterates through the `transform_order` to ensure transformations are applied sequentially.
         - For each transformation, it determines the type and maps it to the corresponding parser class.
-        - Certain transformation types (`combine_columns`, `group`, `find_and_replace`, and `distinct`) 
-          require special handling for their configuration. These are instantiated with a modified 
+        - Certain transformation types (`combine_columns`, `group`, `find_and_replace`, and `distinct`)
+          require special handling for their configuration. These are instantiated with a modified
           configuration structure.
         - Other transformations are instantiated normally with their respective configuration data.
-    
+
         Yields:
-            BaseParser: An instance of the transformation parser for each transformation in the order 
+            BaseParser: An instance of the transformation parser for each transformation in the order
             defined by `transform_order`.
         """
         if self._transforms: