diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 0b08719..825168f 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -2,18 +2,33 @@ name: Unit tests on: pull_request: - types: [opened, synchronize, reopened, ready_for_review] + branches: [main] + push: + branches: [main] permissions: contents: read concurrency: - group: unit-tests-${{ github.event.pull_request.number }} + group: unit-tests-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: + workflow-lint: + name: Validate GitHub workflows + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - name: Check out repository + uses: actions/checkout@v6 + + - name: Run actionlint + uses: raven-actions/actionlint@v2 + validate: name: Ruff and lightweight tests + needs: workflow-lint runs-on: ubuntu-latest timeout-minutes: 30 @@ -25,6 +40,14 @@ jobs: - name: Check out repository uses: actions/checkout@v6 + - name: Cache Conda packages + uses: actions/cache@v4 + with: + path: ~/conda_pkgs_dir + key: conda-${{ runner.os }}-${{ hashFiles('environment.dev.yaml') }} + restore-keys: | + conda-${{ runner.os }}- + - name: Create development environment uses: conda-incubator/setup-miniconda@v4 with: @@ -33,6 +56,15 @@ jobs: miniforge-version: latest conda-solver: libmamba auto-activate-base: false + use-only-tar-bz2: true + + - name: Cache pre-commit environments + uses: actions/cache@v4 + with: + path: ~/.cache/pre-commit + key: pre-commit-${{ runner.os }}-${{ hashFiles('.pre-commit-config.yaml') }} + restore-keys: | + pre-commit-${{ runner.os }}- - name: Show environment run: | diff --git a/.gitignore b/.gitignore index 418eb6d..eab2a29 100644 --- a/.gitignore +++ b/.gitignore @@ -149,8 +149,5 @@ process* # other config-new.yaml -config.all.yaml -config.test.elais.yaml flags_translation_reports/ -test_data/make_collection -.codex \ No newline at end of file +.codex diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8207e40..673bc39 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,9 +1,25 @@ repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-json + - id: check-toml + - id: check-merge-conflict + - id: check-added-large-files + - id: check-case-conflict + - id: detect-private-key + - id: debug-statements + - id: no-commit-to-branch + args: [--branch, main] + - repo: local hooks: - id: ruff-essential - name: Ruff essential checks - entry: ruff check --force-exclude --select=E9,F,B006,B008,B012,B018,B023,B904 + name: Ruff essential checks and import sorting + entry: ruff check --fix --force-exclude --select=E9,F,I,B006,B008,B012,B018,B023,B904 language: system types_or: [python, pyi] exclude: ^notebooks/ diff --git a/LICENSE b/LICENSE index 88422da..6c134e9 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2021 LIneA IT +Copyright (c) 2021 LIneA IT Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 903e6da..09b5f15 100644 --- a/README.md +++ b/README.md @@ -32,13 +32,6 @@ unit tests: pytest ``` -Integration tests execute the complete pipeline, may manage a separate runtime -environment, and are excluded from the default suite. Run them explicitly with: - -```bash -pytest -m slow -``` - ### Pre-commit checks With `pipe_crd_dev` activated, install the repository hook once: @@ -58,13 +51,16 @@ Run the same checks manually across the repository with: pre-commit run --all-files ``` -Pull requests run the same Ruff and lightweight unit-test checks automatically -through GitHub Actions. New commits cancel any older validation run still in -progress for that pull request. +Pull requests targeting `main` and commits merged into `main` run the same Ruff +and lightweight unit-test checks automatically through GitHub Actions. The +workflow also validates its own GitHub Actions configuration. New commits cancel +older validation runs still in progress for the same pull request or branch. ### Test data -This repository currently contains a basic dataset, for testing purposes only. The ideal is to connect the pipelines to systems with access to a larger datasets. +The repository contains three small sample catalogs used by +`config.template.yaml` for local smoke runs. Unit tests create their own +temporary fixtures and do not depend on these files. ### Install @@ -72,18 +68,9 @@ The only requirement is to have `micromamba` available in `PATH`: ```bash git clone https://github.com/linea-it/pzserver_combine_redshift_dedup && cd pzserver_combine_redshift_dedup -./setup.sh -source env.sh -``` - -To install all pipelines at once: - -```bash ./install.sh ``` -The `setup.sh` will suggest a directory where the pipelines and datasets are installed, type 'yes' to confirm or 'no' to configure the desired path in each case with the respective environment variables and then run again `setup.sh`. - The installation script creates the `pipe_crd` environment with `micromamba`. By default the scripts use `MAMBA_ROOT_PREFIX="$HOME/.micromamba"`. On a Slurm cluster, point this variable to a persistent location visible to the jobs if needed: @@ -95,20 +82,9 @@ export MAMBA_ROOT_PREFIX=/path/to/shared/or/persistent/micromamba ## Run a pipeline -To execute, simply: +Copy and edit the example configuration, then execute the pipeline: ```bash -# execute combine redshift catalogs +cp config.template.yaml config.yaml ./run.sh config.yaml process001 ``` - - -## Validation notebook - -To validate your test results, use the notebook `validation.ipynb`: - -``` -conda install -c conda-forge jupyterlab ipykernel -python -m ipykernel install --user --name=pipe_crd -jupyter lab -``` diff --git a/config.py b/config.py index b936c5f..b508efb 100644 --- a/config.py +++ b/config.py @@ -19,9 +19,11 @@ class Instance(BaseModel): job_extra_directives: list[str] = ["--propagate", "--time=12:00:00"] class Scale(BaseModel): - minimum_jobs: int = 10 + minimum_jobs: int = 3 maximum_jobs: int = 22 - worker_recovery_timeout_seconds: float = 600.0 + adaptive_interval_seconds: float = 10.0 + adaptive_scale_down_delay_seconds: float = 180.0 + worker_recovery_timeout_seconds: float = 360.0 worker_recovery_check_interval_seconds: float = 10.0 @model_validator(mode="after") @@ -30,6 +32,12 @@ def validate_limits(self): raise ValueError("minimum_jobs must be at least 1") if self.maximum_jobs < self.minimum_jobs: raise ValueError("maximum_jobs must be >= minimum_jobs") + if self.adaptive_interval_seconds <= 0: + raise ValueError("adaptive_interval_seconds must be positive") + if self.adaptive_scale_down_delay_seconds <= 0: + raise ValueError( + "adaptive_scale_down_delay_seconds must be positive" + ) if self.worker_recovery_timeout_seconds <= 0: raise ValueError("worker_recovery_timeout_seconds must be positive") if self.worker_recovery_check_interval_seconds <= 0: @@ -93,7 +101,7 @@ class Columns(BaseModel): class Param(BaseModel): combine_type: str = "concatenate" extra_columns: dict[str, Any] = Field(default_factory=dict) - # Valid cuts are 1, 2, 3, 4, 5, 6. Invalid values skip the cut with a warning. + # Zero disables the cut; valid active cuts are 1, 2, 3, 4, 5, 6. z_flag_homogenized_value_to_cut: float = 3.0 flags_translation_file: str = str(Path(MAINDIR, "flags_translation.yaml")) insert_DP1_footprint_flag: bool = False diff --git a/config.template.yaml b/config.template.yaml index dad3fc5..dd92a79 100644 --- a/config.template.yaml +++ b/config.template.yaml @@ -23,9 +23,11 @@ executor: # - "--propagate" # - "--time=12:00:00" # scale: -# minimum_jobs: 10 +# minimum_jobs: 3 # maximum_jobs: 22 -# worker_recovery_timeout_seconds: 600 +# adaptive_interval_seconds: 10 +# adaptive_scale_down_delay_seconds: 180 +# worker_recovery_timeout_seconds: 360 # worker_recovery_check_interval_seconds: 10 inputs: @@ -85,7 +87,7 @@ param: # ORIGINAL_ID: # source: id # type: str - z_flag_homogenized_value_to_cut: 0.0 # Valid cuts are 1, 2, 3, 4, 5, 6. Invalid values (<=0, >6, non-integer) skip the cut with a warning. + z_flag_homogenized_value_to_cut: 0.0 # 0 disables the cut; valid active cuts are 1, 2, 3, 4, 5, 6. insert_DP1_footprint_flag: false # Adds is_in_DP1_fields (1/0) insert_rubin_footprint_flag: true # Adds is_in_rubin_footprint (1/0) flags_translation_file: flags_translation.yaml # File with homogenization rules for z_flag and type diff --git a/env.template.sh b/env.template.sh deleted file mode 100644 index f38018f..0000000 --- a/env.template.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -export APP_DIR="" -export PIPELINES_DIR="" -export DATASETS_DIR="" - -export DASK_EXECUTOR_KEY=local diff --git a/flags_translation.yaml b/flags_translation.yaml index 1e0bee5..ba8ab7c 100644 --- a/flags_translation.yaml +++ b/flags_translation.yaml @@ -9,7 +9,11 @@ tiebreaking_priority: # - It exists in all catalogs being crossmatched. # - It is numeric (except for 'instrument_type_homogenized', which uses a string-to-priority mapping). # - Higher values are considered better; lower values are worse. + # Generic priority columns are normalized to float64 per catalog and must + # contain at least one valid numeric value in every input catalog. # The columns will be evaluated in the order listed to resolve duplicates. + # In deduplication modes, z_flag_homogenized is still created and preserved + # for star classification (value 6), even when omitted from this ranking. delta_z_threshold: 0.001 # Maximum allowed redshift difference (|z1 - z2|) to resolve hard ties # Applied only when initial tiebreaking via columns results in a tie (tie_result == 2) @@ -133,7 +137,7 @@ translation_rules: 6: 6 instrument_type_translation: default: "s" - + ASTRODEEP: z_flag_translation: conditions: @@ -149,7 +153,7 @@ translation_rules: - expr: "zspec_survey == '-'" value: "p" default: "p" - + ASTRODEEP-JWST: z_flag_translation: conditions: diff --git a/notebooks/generate_mock_for_fail_conditions.ipynb b/notebooks/generate_mock_for_fail_conditions.ipynb deleted file mode 100644 index 4d907ff..0000000 --- a/notebooks/generate_mock_for_fail_conditions.ipynb +++ /dev/null @@ -1,235 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "409d5a27-a7ba-4ee3-becd-e780228708fa", - "metadata": {}, - "source": [ - "# Generating mocks" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ff258ca5-1f31-4c63-972f-6cf7ee59de45", - "metadata": {}, - "outputs": [], - "source": [ - "#!/usr/bin/env python3\n", - "# -*- coding: utf-8 -*-\n", - "\n", - "\"\"\"\n", - "Generates mock datasets in Parquet format for testing the CRC pipeline.\n", - "\n", - "Generated files:\n", - "1) test_survey_not_supported.parquet\n", - "2) test_homogenized_columns_out_of_range.parquet\n", - "3) test_survey_col_missing.parquet\n", - "4) test_survey_supported_but_wrong_flags.parquet\n", - "\"\"\"\n", - "\n", - "import numpy as np\n", - "import pandas as pd\n", - "\n", - "\n", - "# Number of rows in each dataset\n", - "N_ROWS = 100\n", - "\n", - "# Seed for reproduction (adjust or remove if you want it completely random)\n", - "np.random.seed(42)\n", - "\n", - "\n", - "def random_sky_coordinates(n: int):\n", - " \"\"\"Generate random RA/DEC on the sky (simple uniform in RA and DEC).\"\"\"\n", - " ra = np.random.uniform(0.0, 360.0, n) # RA em graus [0, 360)\n", - " dec = np.random.uniform(-90.0, 90.0, n) # DEC em graus [-90, 90]\n", - " return ra, dec\n", - "\n", - "\n", - "# ============================================================\n", - "# 1) test_survey_not_supported.parquet\n", - "# ------------------------------------------------------------\n", - "# colunas: id, ra, dec, z, z_flag, survey\n", - "# - survey: \"not_supported\"\n", - "# - z_flag: [1, 10]\n", - "# - id: \"1_001\", \"1_002\", ...\n", - "# - z: float em [0, 1]\n", - "# ============================================================\n", - "\n", - "ra1, dec1 = random_sky_coordinates(N_ROWS)\n", - "\n", - "df1 = pd.DataFrame({\n", - " \"id\": [f\"1_{i:03d}\" for i in range(1, N_ROWS + 1)],\n", - " \"ra\": ra1,\n", - " \"dec\": dec1,\n", - " \"z\": np.random.uniform(0.0, 1.0, N_ROWS),\n", - " \"z_flag\": np.random.randint(1, 11, N_ROWS), # 1 a 10 (high é exclusivo)\n", - " \"survey\": [\"not_supported\"] * N_ROWS,\n", - "})\n", - "\n", - "df1.to_parquet(\"test_survey_not_supported.parquet\", index=False)\n", - "\n", - "\n", - "# ============================================================\n", - "# 2) test_homogenized_columns_out_of_range.parquet\n", - "# ------------------------------------------------------------\n", - "# colunas: id, ra, dec, z, z_flag_homogenized,\n", - "# instrument_type_homogenized, survey\n", - "# - survey: \"wrong_homogenized\"\n", - "# - z_flag_homogenized: [1, 10]\n", - "# - id: \"2_001\", \"2_002\", ...\n", - "# - z: float em [0, 1]\n", - "# ============================================================\n", - "\n", - "ra2, dec2 = random_sky_coordinates(N_ROWS)\n", - "\n", - "instrument_types = [\"galaxy\", \"qso\", \"star\"]\n", - "\n", - "df2 = pd.DataFrame({\n", - " \"id\": [f\"2_{i:03d}\" for i in range(1, N_ROWS + 1)],\n", - " \"ra\": ra2,\n", - " \"dec\": dec2,\n", - " \"z\": np.random.uniform(0.0, 1.0, N_ROWS),\n", - " \"z_flag_homogenized\": np.random.randint(1, 11, N_ROWS), # 1 a 10\n", - " \"instrument_type_homogenized\": np.random.choice(instrument_types, N_ROWS),\n", - " \"survey\": [\"wrong_homogenized\"] * N_ROWS,\n", - "})\n", - "\n", - "df2.to_parquet(\"test_homogenized_columns_out_of_range.parquet\", index=False)\n", - "\n", - "\n", - "# ============================================================\n", - "# 3) test_survey_col_missing.parquet\n", - "# ------------------------------------------------------------\n", - "# colunas: id, ra, dec, z, z_flag (SEM survey)\n", - "# - z_flag: [1, 10]\n", - "# - id: \"3_001\", \"3_002\", ...\n", - "# - z: float em [0, 1]\n", - "# ============================================================\n", - "\n", - "ra3, dec3 = random_sky_coordinates(N_ROWS)\n", - "\n", - "df3 = pd.DataFrame({\n", - " \"id\": [f\"3_{i:03d}\" for i in range(1, N_ROWS + 1)],\n", - " \"ra\": ra3,\n", - " \"dec\": dec3,\n", - " \"z\": np.random.uniform(0.0, 1.0, N_ROWS),\n", - " \"z_flag\": np.random.randint(1, 11, N_ROWS), # 1 a 10\n", - " # intencionalmente sem coluna \"survey\"\n", - "})\n", - "\n", - "df3.to_parquet(\"test_survey_col_missing.parquet\", index=False)\n", - "\n", - "\n", - "# ============================================================\n", - "# 4) test_survey_supported_but_wrong_flags.parquet\n", - "# ------------------------------------------------------------\n", - "# colunas: id, ra, dec, z, z_flag, survey\n", - "# - survey: \"2DFGRS\"\n", - "# - z_flag: [10, 20]\n", - "# - id: \"4_001\", \"4_002\", ...\n", - "# - z: float em [0, 1]\n", - "# ============================================================\n", - "\n", - "ra4, dec4 = random_sky_coordinates(N_ROWS)\n", - "\n", - "df4 = pd.DataFrame({\n", - " \"id\": [f\"4_{i:03d}\" for i in range(1, N_ROWS + 1)],\n", - " \"ra\": ra4,\n", - " \"dec\": dec4,\n", - " \"z\": np.random.uniform(0.0, 1.0, N_ROWS),\n", - " \"z_flag\": np.random.randint(10, 21, N_ROWS), # 10 a 20\n", - " \"survey\": [\"2DFGRS\"] * N_ROWS,\n", - "})\n", - "\n", - "df4.to_parquet(\"test_survey_supported_but_wrong_flags.parquet\", index=False)\n", - "\n", - "\n", - "print(\"Arquivos Parquet gerados com sucesso!\")" - ] - }, - { - "cell_type": "markdown", - "id": "c7f9ad98-c710-45a8-b72d-73d283c98969", - "metadata": {}, - "source": [ - "# Validation" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "121a0373-ea5a-49b0-93e3-6327b5e3f26f", - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d5e26a14-062e-4ca7-a288-df4f2b880651", - "metadata": {}, - "outputs": [], - "source": [ - "df_1 = pd.read_parquet(\"test_survey_not_supported.parquet\")\n", - "df_1.head()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8516b4d3-7498-4b2a-8f96-c637cfce3941", - "metadata": {}, - "outputs": [], - "source": [ - "df_2 = pd.read_parquet(\"test_homogenized_columns_out_of_range.parquet\")\n", - "df_2.head()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "daed2bbb-4822-419c-ba16-9b9440994181", - "metadata": {}, - "outputs": [], - "source": [ - "df_3 = pd.read_parquet(\"test_survey_col_missing.parquet\")\n", - "df_3.head()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2b000317-94a3-485a-ba56-d1d2612f61b0", - "metadata": {}, - "outputs": [], - "source": [ - "df_4 = pd.read_parquet(\"test_survey_supported_but_wrong_flags.parquet\")\n", - "df_4.head()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "CONDA_ENV_data_preparation", - "language": "python", - "name": "conda_env_data_preparation" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.13.7" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks/generate_mock_old.ipynb b/notebooks/generate_mock_old.ipynb deleted file mode 100644 index 55adf4a..0000000 --- a/notebooks/generate_mock_old.ipynb +++ /dev/null @@ -1,344 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "6226ec65-7ffe-4bf2-a937-ef890c222563", - "metadata": {}, - "source": [ - "# Generating mock data for CRC" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d0623949-74db-46fc-8a2f-9877991f01d9", - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "import numpy as np\n", - "import os\n", - "\n", - "# Ensure output directory exists\n", - "os.makedirs(\"data\", exist_ok=True)\n", - "\n", - "rng = np.random.default_rng(seed=42)\n", - "\n", - "# =============================\n", - "# Helper functions\n", - "# =============================\n", - "\n", - "def jitter_coords(base_ra, base_dec, n, jitter=0.0001):\n", - " return (\n", - " base_ra + rng.uniform(-jitter, jitter, size=n),\n", - " base_dec + rng.uniform(-jitter, jitter, size=n)\n", - " )\n", - "\n", - "def generate_unique_sources(n, ra_range=(0, 360), dec_range=(-30, 30)):\n", - " ra = rng.uniform(*ra_range, size=n)\n", - " dec = rng.uniform(*dec_range, size=n)\n", - " return ra, dec\n", - "\n", - "def generate_redshifts(n):\n", - " return rng.uniform(0.01, 1.0, size=n)\n", - "\n", - "def generate_redshifts_with_jitter(base_z, low_jitter=0.0003, high_jitter=0.005, frac_low=0.8):\n", - " n = len(base_z)\n", - " is_low = rng.random(n) < frac_low\n", - " jitter = np.where(is_low,\n", - " rng.uniform(-low_jitter, low_jitter, n),\n", - " rng.uniform(-high_jitter, high_jitter, n))\n", - " return base_z + jitter\n", - "\n", - "def generate_primus_flags(n):\n", - " # 60% flag 4, 30% flag 3, 10% flag 2 or -1\n", - " flags = rng.choice(\n", - " [4, 3, 2, -1],\n", - " size=n,\n", - " p=[0.60, 0.30, 0.05, 0.05] # 2 + (-1) = 10%\n", - " )\n", - " return flags\n", - "\n", - "def generate_2dflens_flags(n):\n", - " # 60% flag 4, 30% flag 3, 8% flag 2 or 1, 2% flag 6\n", - " flags = rng.choice(\n", - " [4, 3, 2, 1, 6],\n", - " size=n,\n", - " p=[0.60, 0.30, 0.04, 0.04, 0.02]\n", - " )\n", - " return flags\n", - "\n", - "def generate_2mrs_zerr(n):\n", - " return rng.uniform(0.00001, 0.001, size=n)\n", - "\n", - "def build_catalog(id_prefix, ra, dec, z, suffix, z_flag=None, z_err=None, survey=None):\n", - " size = len(ra)\n", - " data = {\n", - " f\"id_{suffix}\": [f\"{id_prefix}_{i}\" for i in range(size)],\n", - " f\"ra_{suffix}\": ra,\n", - " f\"dec_{suffix}\": dec,\n", - " f\"z_{suffix}\": z,\n", - " f\"survey_{suffix}\": survey,\n", - " f\"random_{suffix}\": rng.uniform(0, 1, size)\n", - " }\n", - " if z_flag is not None:\n", - " data[f\"z_flag_{suffix}\"] = z_flag\n", - " if z_err is not None:\n", - " data[f\"z_err_{suffix}\"] = z_err\n", - " return pd.DataFrame(data)\n", - "\n", - "def generate_exact_selfmatch_coords(n_pairs):\n", - " ra, dec = generate_unique_sources(n_pairs)\n", - " ra_duplicated = np.repeat(ra, 2)\n", - " dec_duplicated = np.repeat(dec, 2)\n", - " return ra_duplicated, dec_duplicated\n", - "\n", - "# =============================\n", - "# Generate matched & unique sources\n", - "# =============================\n", - "\n", - "# Match among all 3 catalogs (10)\n", - "base_ra_123, base_dec_123 = generate_unique_sources(10)\n", - "ra1_123, dec1_123 = jitter_coords(base_ra_123, base_dec_123, 10)\n", - "ra2_123, dec2_123 = jitter_coords(base_ra_123, base_dec_123, 10)\n", - "ra3_123, dec3_123 = jitter_coords(base_ra_123, base_dec_123, 10)\n", - "\n", - "# Match between mock1 and mock2 (20)\n", - "base_ra_12, base_dec_12 = generate_unique_sources(20)\n", - "ra1_12, dec1_12 = jitter_coords(base_ra_12, base_dec_12, 20)\n", - "ra2_12, dec2_12 = jitter_coords(base_ra_12, base_dec_12, 20)\n", - "\n", - "# Match between mock2 and mock3 (30)\n", - "base_ra_23, base_dec_23 = generate_unique_sources(30)\n", - "ra2_23, dec2_23 = jitter_coords(base_ra_23, base_dec_23, 30)\n", - "ra3_23, dec3_23 = jitter_coords(base_ra_23, base_dec_23, 30)\n", - "\n", - "# Match between mock1 and mock3 (40)\n", - "base_ra_13, base_dec_13 = generate_unique_sources(40)\n", - "ra1_13, dec1_13 = jitter_coords(base_ra_13, base_dec_13, 40)\n", - "ra3_13, dec3_13 = jitter_coords(base_ra_13, base_dec_13, 40)\n", - "\n", - "# Internal duplicates (50 per catalog = 25 pairs × 2)\n", - "ra1_self, dec1_self = generate_exact_selfmatch_coords(25)\n", - "ra2_self, dec2_self = generate_exact_selfmatch_coords(25)\n", - "ra3_self, dec3_self = generate_exact_selfmatch_coords(25)\n", - "\n", - "# Unique sources (950 per catalog)\n", - "ra1_unique, dec1_unique = generate_unique_sources(950)\n", - "ra2_unique, dec2_unique = generate_unique_sources(950)\n", - "ra3_unique, dec3_unique = generate_unique_sources(950)\n", - "\n", - "# =============================\n", - "# Generate base z for matched objects\n", - "# =============================\n", - "z_123 = generate_redshifts(10)\n", - "z_12 = generate_redshifts(20)\n", - "z_23 = generate_redshifts(30)\n", - "z_13 = generate_redshifts(40)\n", - "z_self1 = generate_redshifts(25)\n", - "z_self2 = generate_redshifts(25)\n", - "z_self3 = generate_redshifts(25)\n", - "\n", - "# =============================\n", - "# Build mock catalogs\n", - "# =============================\n", - "\n", - "def make_mock1():\n", - " z = np.concatenate([\n", - " generate_redshifts_with_jitter(z_123),\n", - " generate_redshifts_with_jitter(z_12),\n", - " generate_redshifts_with_jitter(z_13),\n", - " np.repeat(z_self1, 2),\n", - " generate_redshifts(950)\n", - " ])\n", - " ra = np.concatenate([ra1_123, ra1_12, ra1_13, ra1_self, ra1_unique])\n", - " dec = np.concatenate([dec1_123, dec1_12, dec1_13, dec1_self, dec1_unique])\n", - " z_flag = generate_primus_flags(len(ra)) \n", - " return build_catalog(\"mock1\", ra, dec, z, suffix=\"1\", z_flag=z_flag, survey=\"PRIMUS\")\n", - "\n", - "def make_mock2():\n", - " z = np.concatenate([\n", - " generate_redshifts_with_jitter(z_123),\n", - " generate_redshifts_with_jitter(z_12),\n", - " generate_redshifts_with_jitter(z_23),\n", - " np.repeat(z_self2, 2),\n", - " generate_redshifts(950)\n", - " ])\n", - " ra = np.concatenate([ra2_123, ra2_12, ra2_23, ra2_self, ra2_unique])\n", - " dec = np.concatenate([dec2_123, dec2_12, dec2_23, dec2_self, dec2_unique])\n", - " z_flag = generate_2dflens_flags(len(ra))\n", - " return build_catalog(\"mock2\", ra, dec, z, suffix=\"2\", z_flag=z_flag, survey=\"2DFLENS\")\n", - "\n", - "def make_mock3():\n", - " z = np.concatenate([\n", - " generate_redshifts_with_jitter(z_123),\n", - " generate_redshifts_with_jitter(z_23),\n", - " generate_redshifts_with_jitter(z_13),\n", - " np.repeat(z_self3, 2),\n", - " generate_redshifts(950)\n", - " ])\n", - " ra = np.concatenate([ra3_123, ra3_23, ra3_13, ra3_self, ra3_unique])\n", - " dec = np.concatenate([dec3_123, dec3_23, dec3_13, dec3_self, dec3_unique])\n", - " z_err = generate_2mrs_zerr(len(ra))\n", - " return build_catalog(\"mock3\", ra, dec, z, suffix=\"3\", z_err=z_err, survey=\"2MRS\")\n", - "\n", - "# =============================\n", - "# Save to Parquet\n", - "# =============================\n", - "mock1 = make_mock1()\n", - "mock2 = make_mock2()\n", - "mock3 = make_mock3()\n", - "\n", - "mock1.to_parquet(\"data/mock1.parquet\", index=False)\n", - "mock2.to_parquet(\"data/mock2.parquet\", index=False)\n", - "mock3.to_parquet(\"data/mock3.parquet\", index=False)\n", - "\n", - "# Optional preview\n", - "mock1.head(2), mock2.head(2), mock3.head(2)" - ] - }, - { - "cell_type": "markdown", - "id": "4bf937ea-c3a5-4034-b04a-cbce00bbea97", - "metadata": {}, - "source": [ - "# Validation" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c67f773b-2d5a-44eb-8f3e-1ad1223b0df4", - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "from astropy.coordinates import SkyCoord\n", - "from astropy import units as u\n", - "\n", - "# =============================\n", - "# 1. Load catalogs\n", - "# =============================\n", - "mock1 = pd.read_parquet(\"data/mock1.parquet\")\n", - "mock2 = pd.read_parquet(\"data/mock2.parquet\")\n", - "mock3 = pd.read_parquet(\"data/mock3.parquet\")\n", - "\n", - "# =============================\n", - "# 2. Create SkyCoord objects\n", - "# =============================\n", - "coords1 = SkyCoord(ra=mock1[\"ra_1\"].values * u.deg, dec=mock1[\"dec_1\"].values * u.deg)\n", - "coords2 = SkyCoord(ra=mock2[\"ra_2\"].values * u.deg, dec=mock2[\"dec_2\"].values * u.deg)\n", - "coords3 = SkyCoord(ra=mock3[\"ra_3\"].values * u.deg, dec=mock3[\"dec_3\"].values * u.deg)\n", - "\n", - "# =============================\n", - "# 3. Crossmatching between catalogs\n", - "# =============================\n", - "radius = 1.0 / 3600 # 1 arcsecond in degrees\n", - "\n", - "# mock1 → mock2\n", - "idx1_2, sep1_2, _ = coords1.match_to_catalog_sky(coords2)\n", - "matches12 = sep1_2.deg < radius\n", - "\n", - "# mock2 → mock3\n", - "idx2_3, sep2_3, _ = coords2.match_to_catalog_sky(coords3)\n", - "matches23 = sep2_3.deg < radius\n", - "\n", - "# mock1 → mock3\n", - "idx1_3, sep1_3, _ = coords1.match_to_catalog_sky(coords3)\n", - "matches13 = sep1_3.deg < radius\n", - "\n", - "# For exclusive matches: avoid shape mismatch\n", - "# mock2 → mock1 (reverse of matches12)\n", - "idx2_1, sep2_1, _ = coords2.match_to_catalog_sky(coords1)\n", - "reverse_matches21 = sep2_1.deg < radius\n", - "\n", - "# =============================\n", - "# 4. Match classification\n", - "# =============================\n", - "\n", - "# Triple match: objects in mock1 matching both mock2 and mock3\n", - "triplet = matches12 & matches13\n", - "\n", - "# Exclusive pairs (not matched with the third catalog)\n", - "pair_12_only = matches12 & ~matches13\n", - "pair_23_only = matches23 & ~reverse_matches21\n", - "pair_13_only = matches13 & ~matches12\n", - "\n", - "# =============================\n", - "# 5. Internal duplicates (identical RA/DEC within the same catalog)\n", - "# =============================\n", - "internal1 = mock1.duplicated(subset=[\"ra_1\", \"dec_1\"], keep=False)\n", - "internal2 = mock2.duplicated(subset=[\"ra_2\", \"dec_2\"], keep=False)\n", - "internal3 = mock3.duplicated(subset=[\"ra_3\", \"dec_3\"], keep=False)\n", - "\n", - "# =============================\n", - "# 6. Unmatched objects (no external match)\n", - "# =============================\n", - "no_match1 = ~matches12 & ~matches13\n", - "no_match2 = ~matches23 & ~reverse_matches21\n", - "no_match3 = ~reverse_matches21 & ~matches23\n", - "\n", - "# =============================\n", - "# 7. Print results\n", - "# =============================\n", - "print(\"🔗 Triple match (1 ∩ 2 ∩ 3):\", triplet.sum())\n", - "print(\"🔗 Pair match only 1 ∩ 2:\", pair_12_only.sum())\n", - "print(\"🔗 Pair match only 2 ∩ 3:\", pair_23_only.sum())\n", - "print(\"🔗 Pair match only 1 ∩ 3:\", pair_13_only.sum())\n", - "print()\n", - "print(\"♻️ Internal duplicates:\")\n", - "print(\" - mock1:\", internal1.sum())\n", - "print(\" - mock2:\", internal2.sum())\n", - "print(\" - mock3:\", internal3.sum())\n", - "print()\n", - "print(\"🧩 Objects with no external match:\")\n", - "print(\" - mock1:\", no_match1.sum())\n", - "print(\" - mock2:\", no_match2.sum())\n", - "print(\" - mock3:\", no_match3.sum())" - ] - }, - { - "cell_type": "markdown", - "id": "cdbccba3-f334-4a6d-9222-187a4169acb4", - "metadata": {}, - "source": [ - "# Spatial Distribution" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "33668ffe-a656-48af-b173-ff09441d4fbf", - "metadata": {}, - "outputs": [], - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "plt.scatter(mock1[\"ra_1\"], mock1[\"dec_1\"])\n", - "plt.scatter(mock2[\"ra_2\"], mock2[\"dec_2\"])\n", - "plt.scatter(mock3[\"ra_3\"], mock3[\"dec_3\"])" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "pipe_csc", - "language": "python", - "name": "pipe_csc" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.11" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks/generate_random_samples.ipynb b/notebooks/generate_random_samples.ipynb deleted file mode 100644 index 76fe961..0000000 --- a/notebooks/generate_random_samples.ipynb +++ /dev/null @@ -1,244 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "99edb3dc-03b4-4378-8496-d910211ff804", - "metadata": {}, - "source": [ - "# Generating random samples for many files at once" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ae52ee47-d0de-4033-95ab-465c7517af57", - "metadata": {}, - "outputs": [], - "source": [ - "from __future__ import annotations\n", - "import random\n", - "from pathlib import Path\n", - "import pyarrow as pa\n", - "import pyarrow.parquet as pq\n", - "\n", - "# ----------------- CONFIG -----------------\n", - "# Directories containing your input Parquet files\n", - "INPUT_DIRS = [\n", - " \"/scratch/users/luigi.silva/speczs-catalogs/processed\",\n", - " \"/scratch/users/luigi.silva/speczs-catalogs/johns-catalogs\",\n", - "]\n", - "\n", - "# Directory where the sampled files will be saved\n", - "OUTPUT_DIR = Path(\"/scratch/users/luigi.silva/pzserver_pipelines/combine_redshift_dedup/test_data\")\n", - "\n", - "# Maximum number of rows per sample\n", - "SAMPLE_MAX = 1000\n", - "\n", - "# Random seed for reproducibility\n", - "SEED = 42\n", - "# ------------------------------------------\n", - "\n", - "\n", - "def ensure_outdir(path: Path) -> None:\n", - " \"\"\"Make sure the output directory exists.\"\"\"\n", - " path.mkdir(parents=True, exist_ok=True)\n", - "\n", - "\n", - "def sample_parquet_reservoir(in_path: Path, k: int, seed: int = 42) -> pa.Table:\n", - " \"\"\"\n", - " Perform random sampling (without replacement) from a Parquet file.\n", - " - If the file has <= k rows, return all rows.\n", - " - Otherwise, use reservoir sampling to select k rows while reading in streaming mode.\n", - " This avoids loading the entire file into memory.\n", - " \"\"\"\n", - " random.seed(seed)\n", - "\n", - " pf = pq.ParquetFile(str(in_path))\n", - " total_rows = pf.metadata.num_rows\n", - "\n", - " # Case 1: small file -> just read all rows\n", - " if total_rows <= k:\n", - " return pf.read()\n", - "\n", - " # Case 2: large file -> reservoir sampling\n", - " schema = pf.schema_arrow\n", - " columns = [[] for _ in schema] # temporary storage for sampled rows\n", - "\n", - " seen = 0 # number of rows processed so far\n", - "\n", - " for batch in pf.iter_batches():\n", - " # Get columns as arrays for easier row access\n", - " cols = [batch.column(i) for i in range(batch.num_columns)]\n", - " n = batch.num_rows\n", - "\n", - " for i in range(n):\n", - " if seen < k:\n", - " # Fill the reservoir until it reaches size k\n", - " for c_idx, arr in enumerate(cols):\n", - " columns[c_idx].append(arr[i].as_py())\n", - " else:\n", - " # Replace elements with decreasing probability\n", - " j = random.randint(0, seen)\n", - " if j < k:\n", - " for c_idx, arr in enumerate(cols):\n", - " columns[c_idx][j] = arr[i].as_py()\n", - " seen += 1\n", - "\n", - " # Convert sampled Python lists back to Arrow arrays\n", - " arrays = [pa.array(col, type=schema.field(i).type) for i, col in enumerate(columns)]\n", - " table = pa.Table.from_arrays(arrays, names=[f.name for f in schema])\n", - "\n", - " assert len(table) == min(k, total_rows)\n", - " return table\n", - "\n", - "\n", - "# ----------------- MAIN EXECUTION -----------------\n", - "ensure_outdir(OUTPUT_DIR)\n", - "count = 0\n", - "\n", - "for d in INPUT_DIRS:\n", - " in_dir = Path(d)\n", - " if not in_dir.is_dir():\n", - " print(f\"[WARN] Directory not found: {in_dir}\")\n", - " continue\n", - "\n", - " for p in sorted(in_dir.glob(\"*.parquet\")):\n", - " survey_name = p.stem # file name without extension\n", - " out_path = OUTPUT_DIR / f\"{survey_name}_random_sample.parquet\"\n", - "\n", - " print(f\"[INFO] Sampling {p} -> {out_path}\")\n", - " tbl = sample_parquet_reservoir(p, SAMPLE_MAX, seed=SEED)\n", - "\n", - " pq.write_table(tbl, out_path)\n", - " count += 1\n", - "\n", - "print(f\"[DONE] {count} files processed. Output in: {OUTPUT_DIR}\")" - ] - }, - { - "cell_type": "markdown", - "id": "0b2e893d-2203-4741-990c-215b0a2e6f81", - "metadata": {}, - "source": [ - "# Generating random sample for a single file" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c4cf3a2d-7878-4fb5-8ec8-328e89022bf7", - "metadata": {}, - "outputs": [], - "source": [ - "from __future__ import annotations\n", - "import random\n", - "from pathlib import Path\n", - "import pyarrow as pa\n", - "import pyarrow.parquet as pq\n", - "\n", - "# --------------- CONFIG ---------------\n", - "# Single input Parquet file (extension \"parqut\" is fine as long as the file exists and is Parquet)\n", - "INPUT_FILE = Path(\"/scratch/users/luigi.silva/pzserver_pipelines/combine_redshift_dedup/process001/crd.parquet\")\n", - "\n", - "# Exact output path\n", - "OUTPUT_PATH = Path(\"./test_data/pipeline_generated_sample.parquet\")\n", - "\n", - "# Maximum number of rows in the sample\n", - "SAMPLE_MAX = 1000\n", - "\n", - "# Random seed for reproducibility\n", - "SEED = 42\n", - "# --------------------------------------\n", - "\n", - "\n", - "def sample_parquet_reservoir(in_path: Path, k: int, seed: int = 42) -> pa.Table:\n", - " \"\"\"\n", - " Return a random sample (without replacement) of up to k rows from a Parquet file.\n", - " - If the file has <= k rows, return the whole file.\n", - " - Otherwise, use reservoir sampling while streaming through the file in row batches\n", - " (avoids loading the entire file into memory).\n", - " \"\"\"\n", - " random.seed(seed)\n", - "\n", - " pf = pq.ParquetFile(str(in_path))\n", - " total_rows = pf.metadata.num_rows\n", - " schema = pf.schema_arrow\n", - "\n", - " # Handle empty files explicitly\n", - " if total_rows == 0:\n", - " arrays = [pa.array([], type=schema.field(i).type) for i in range(len(schema))]\n", - " return pa.Table.from_arrays(arrays, names=[f.name for f in schema])\n", - "\n", - " # Small file -> just read all rows\n", - " if total_rows <= k:\n", - " return pf.read()\n", - "\n", - " # Large file -> reservoir sampling\n", - " columns = [[] for _ in range(len(schema))] # temporary storage for sampled rows\n", - " seen = 0 # number of rows processed so far\n", - "\n", - " for batch in pf.iter_batches():\n", - " # Access columns as Arrow arrays for quick row access\n", - " cols = [batch.column(i) for i in range(batch.num_columns)]\n", - " n = batch.num_rows\n", - "\n", - " for i in range(n):\n", - " if seen < k:\n", - " # Fill the reservoir until it reaches size k\n", - " for c_idx, arr in enumerate(cols):\n", - " columns[c_idx].append(arr[i].as_py())\n", - " else:\n", - " # Replace elements with decreasing probability\n", - " j = random.randint(0, seen)\n", - " if j < k:\n", - " for c_idx, arr in enumerate(cols):\n", - " columns[c_idx][j] = arr[i].as_py()\n", - " seen += 1\n", - "\n", - " # Convert sampled Python lists back to Arrow arrays with the original schema types\n", - " arrays = [pa.array(col, type=schema.field(i).type) for i, col in enumerate(columns)]\n", - " table = pa.Table.from_arrays(arrays, names=[f.name for f in schema])\n", - "\n", - " assert len(table) == min(k, total_rows), \"Sample size mismatch\"\n", - " return table\n", - "\n", - "\n", - "def main():\n", - " if not INPUT_FILE.is_file():\n", - " raise FileNotFoundError(f\"Input Parquet not found: {INPUT_FILE}\")\n", - "\n", - " OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True)\n", - "\n", - " print(f\"[INFO] Sampling {INPUT_FILE} -> {OUTPUT_PATH}\")\n", - " tbl = sample_parquet_reservoir(INPUT_FILE, SAMPLE_MAX, seed=SEED)\n", - " pq.write_table(tbl, OUTPUT_PATH)\n", - " print(f\"[DONE] {len(tbl)} rows written to: {OUTPUT_PATH}\")\n", - "\n", - "\n", - "if __name__ == \"__main__\":\n", - " main()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "data_preparation", - "language": "python", - "name": "data_preparation" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.10" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks/validation.ipynb b/notebooks/validation.ipynb deleted file mode 100644 index 835a49a..0000000 --- a/notebooks/validation.ipynb +++ /dev/null @@ -1,1146 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "06d2c6aa-e836-4c37-b67f-b8a828c80590", - "metadata": {}, - "source": [ - "# Imports" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "351b9448-1cdf-46a1-b07c-b031c3c5f12c", - "metadata": {}, - "outputs": [], - "source": [ - "# =====================\n", - "# Built-in\n", - "# =====================\n", - "import os\n", - "import glob\n", - "import warnings\n", - "import sys\n", - "import importlib.util\n", - "from pathlib import Path\n", - "import getpass\n", - "import io\n", - "import csv\n", - "\n", - "# =====================\n", - "# Third-party\n", - "# =====================\n", - "import numpy as np\n", - "import pandas as pd\n", - "import pyarrow.parquet as pq\n", - "from IPython.display import display\n", - "\n", - "# =====================\n", - "# Local module (via importlib)\n", - "# =====================\n", - "mod_path = Path(\"../packages/product_handle.py\").resolve()\n", - "spec = importlib.util.spec_from_file_location(\"product_handle\", mod_path)\n", - "product_handle = importlib.util.module_from_spec(spec)\n", - "sys.modules[\"product_handle\"] = product_handle\n", - "spec.loader.exec_module(product_handle)\n", - "\n", - "ProductHandle = product_handle.ProductHandle\n", - "\n", - "# =====================\n", - "# User setup\n", - "# =====================\n", - "user = getpass.getuser()" - ] - }, - { - "cell_type": "markdown", - "id": "921fa07e-c9e5-4ff5-bfa9-ff27115717d6", - "metadata": {}, - "source": [ - "# Configuration" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5c66b946-64d0-4182-aca2-1268f6b3a2e1", - "metadata": {}, - "outputs": [], - "source": [ - "# =========================================================\n", - "# CONFIGURATION SECTION\n", - "# =========================================================\n", - "#input_paths = [\n", - "# f\"/scratch/users/{user}/speczs-catalogs/processed/2dfgrs_final_release.parquet\",\n", - "# f\"/scratch/users/{user}/speczs-catalogs/processed/2dflens_final_release.parquet\",\n", - "# f\"/scratch/users/{user}/speczs-catalogs/processed/2mrs_v240.parquet\",\n", - "# f\"/scratch/users/{user}/speczs-catalogs/processed/3dhst_v4.1.5.parquet\",\n", - "# f\"/scratch/users/{user}/speczs-catalogs/processed/6dfgs_dr3.parquet\",\n", - "# f\"/scratch/users/{user}/speczs-catalogs/processed/astrodeep_jwst.parquet\",\n", - "# f\"/scratch/users/luigi.silva/speczs-catalogs/processed/astrodeep-gs43.parquet\",\n", - "# f\"/scratch/users/{user}/speczs-catalogs/processed/desi_dr1_in_lsst_dp1_fields.parquet\",\n", - "# f\"/scratch/users/{user}/speczs-catalogs/processed/jades_dr3.parquet\",\n", - "# f\"/scratch/users/{user}/speczs-catalogs/processed/mosdef_final_release.parquet\",\n", - "# f\"/scratch/users/{user}/speczs-catalogs/processed/ozdes_dr2.parquet\",\n", - "# f\"/scratch/users/{user}/speczs-catalogs/processed/primus_dr1.parquet\",\n", - "# f\"/scratch/users/{user}/speczs-catalogs/processed/vandels_dr4.parquet\",\n", - "# f\"/scratch/users/{user}/speczs-catalogs/processed/vlt_vimos_v2.0.1.parquet\",\n", - "# f\"/scratch/users/{user}/speczs-catalogs/processed/vuds_dr1.parquet\",\n", - "# f\"/scratch/users/{user}/speczs-catalogs/processed/vvds_final_release.parquet\",\n", - "# f\"/scratch/users/{user}/speczs-catalogs/johns-catalogs/z_cat_CANDELS_clean_sitcomtn-154.parquet\",\n", - "# f\"/scratch/users/{user}/speczs-catalogs/johns-catalogs/z_cat_NED_clean_sitcomtn-154.parquet\",\n", - "# f\"/scratch/users/{user}/pzserver_pipelines/combine_redshift_dedup/test_data/pipeline_generated_sample.parquet\"\n", - "#]\n", - "\n", - "input_paths = glob.glob('../test_data/*sample.parquet')\n", - "\n", - "final_catalog_files = glob.glob(\"../process001/crd.*\")\n", - "final_catalog_path = final_catalog_files[0]\n", - "prepared_temp_dir = f\"./process001/temp/\"\n", - "\n", - "combine_mode = \"concatenate_and_mark_duplicates\" # Options: \"concatenate\", \"concatenate_and_mark_duplicates\", or \"concatenate_and_remove_duplicates\"" - ] - }, - { - "cell_type": "markdown", - "id": "02b1390e-4d78-46dc-a879-93741200e42d", - "metadata": {}, - "source": [ - "# Validation" - ] - }, - { - "cell_type": "markdown", - "id": "6f9d742f-2815-4c78-b471-ed538be842b7", - "metadata": {}, - "source": [ - "## Basic Info" - ] - }, - { - "cell_type": "markdown", - "id": "5db2c2af-86a5-4c2c-abfb-3f304ed3b90d", - "metadata": {}, - "source": [ - "Counting input and output rows." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5ae0ff59-ef63-4ff9-886f-37606a01cb68", - "metadata": {}, - "outputs": [], - "source": [ - "# ========================================================\n", - "# COUNT INPUT ROWS\n", - "# ========================================================\n", - "total_input_rows = 0\n", - "for path in input_paths:\n", - " if os.path.exists(path):\n", - " try:\n", - " ph = ProductHandle(path)\n", - " ddf = ph.to_ddf()\n", - " n_rows = ddf.shape[0].compute()\n", - " print(f\"{path} -> {n_rows} rows\")\n", - " total_input_rows += n_rows\n", - " except Exception as e:\n", - " warnings.warn(f\"⚠️ Could not read {path}: {e}\")\n", - " else:\n", - " warnings.warn(f\"⚠️ File not found: {path}\")\n", - "\n", - "print(f\"✅ Total number of input rows: {total_input_rows}\")\n", - "\n", - "# =========================================================\n", - "# LOAD FINAL MERGED CATALOG\n", - "# =========================================================\n", - "if not os.path.exists(final_catalog_path):\n", - " raise FileNotFoundError(f\"❌ Final catalog not found: {final_catalog_path}\")\n", - "\n", - "ph_final = ProductHandle(final_catalog_path)\n", - "df_final = ph_final.to_ddf().compute() \n", - "print(f\"✅ Total number of rows in final catalog: {len(df_final)}\")" - ] - }, - { - "cell_type": "markdown", - "id": "5d55c9e8-c4e8-4c7b-a057-8f3fd80471c0", - "metadata": {}, - "source": [ - "Printing the dataframe." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "46685e01-a0c4-4da7-b5cf-3e7a43794889", - "metadata": {}, - "outputs": [], - "source": [ - "df_final" - ] - }, - { - "cell_type": "markdown", - "id": "ed728553-feda-4264-aee8-482e202f6443", - "metadata": {}, - "source": [ - "Dataframe columns." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "fc6a85f2-2deb-419b-84f0-893ccbf4ce3b", - "metadata": {}, - "outputs": [], - "source": [ - "df_final.columns" - ] - }, - { - "cell_type": "markdown", - "id": "2c4c7fcd-64de-489a-8254-f83e08a2abbc", - "metadata": {}, - "source": [ - "Dataframe columns types." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8ba9d0d0-c23e-41d4-a01c-3e53161f01eb", - "metadata": {}, - "outputs": [], - "source": [ - "df_final.dtypes" - ] - }, - { - "cell_type": "markdown", - "id": "1db02aa8-5dea-41c3-9807-a18308f4b93e", - "metadata": {}, - "source": [ - "Basic statistics." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cfff02fa-50be-4d05-ae3b-d62ae76df68f", - "metadata": {}, - "outputs": [], - "source": [ - "if combine_mode != \"concatenate\":\n", - " cols_to_drop = [c for c in df_final.columns if c.startswith(\"group_id\")]\n", - " display(df_final.drop(columns=cols_to_drop).describe())\n", - "else:\n", - " display(df_final.describe())" - ] - }, - { - "cell_type": "markdown", - "id": "332f2bdc-55a9-463c-859b-9dbb1ecd1e57", - "metadata": {}, - "source": [ - "Counting tie_result values." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cf326961-1cf0-4587-a47c-bbec4a99dc76", - "metadata": {}, - "outputs": [], - "source": [ - "if combine_mode != \"concatenate\":\n", - " print(df_final[\"tie_result\"].value_counts())" - ] - }, - { - "cell_type": "markdown", - "id": "d3cd6ce4-6d28-484f-bf65-68b44085a852", - "metadata": {}, - "source": [ - "Counting source values." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a9a2a90c-386c-4904-824a-547509576407", - "metadata": {}, - "outputs": [], - "source": [ - "df_final[\"source\"].value_counts()" - ] - }, - { - "cell_type": "markdown", - "id": "e99e8dd0-f6cc-4310-a896-fba216515967", - "metadata": {}, - "source": [ - "Checking the percentage of unsolved objects." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "818d2df1-3df5-45a3-a1bb-217a2925c591", - "metadata": {}, - "outputs": [], - "source": [ - "if combine_mode != \"concatenate\":\n", - " # Total number of objects\n", - " total_all = len(df_final)\n", - " \n", - " # Filter objects that were compared (compared_to is not null or empty)\n", - " if \"compared_to\" in df_final.columns.to_list():\n", - " mask_compared = df_final[\"compared_to\"].notna() & (df_final[\"compared_to\"] != \"\")\n", - " df_compared = df_final[mask_compared]\n", - " \n", - " # Count how many have tie_result == 2\n", - " count_tie2 = (df_final[\"tie_result\"] == 2).sum()\n", - " count_tie2_compared = (df_compared[\"tie_result\"] == 2).sum()\n", - " \n", - " # Percentages\n", - " percent_all = (count_tie2 / total_all) * 100 if total_all > 0 else 0\n", - " percent_compared = (count_tie2_compared / len(df_compared)) * 100 if len(df_compared) > 0 else 0\n", - " \n", - " # Formatted print\n", - " print(f\"📊 tie_result == 2 represents:\")\n", - " print(f\" • {percent_all:.2f}% of the total ({count_tie2} out of {total_all})\")\n", - " print(f\" • {percent_compared:.2f}% of the compared objects ({count_tie2_compared} out of {len(df_compared)})\")" - ] - }, - { - "cell_type": "markdown", - "id": "19122dc4-9a55-41ff-8c00-2ab72eda2b89", - "metadata": {}, - "source": [ - "## Individual Catalogs Deduplication Validation" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2b87f693-072b-48ea-bf78-e7fad7f351c4", - "metadata": {}, - "outputs": [], - "source": [ - "if combine_mode != \"concatenate\":\n", - " if \"test_data/pipeline_generated_sample.parquet\" in input_paths:\n", - " df_final_val = df_final[df_final[\"source\"] != \"019_pipeline_sample\"]\n", - " else:\n", - " df_final_val = df_final" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "74320729-c712-4a7a-88ce-549891c2b792", - "metadata": {}, - "outputs": [], - "source": [ - "if combine_mode != \"concatenate\":\n", - " %load_ext autoreload\n", - " %autoreload 2\n", - " \n", - " import validation_functions as vf" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "41e7d96d-f5e5-4368-b9db-99e8fa689376", - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "if combine_mode != \"concatenate\":\n", - " if \"compared_to\" in df_final.columns.to_list():\n", - " res = vf.validate_intra_source_cells_fast(df_final_val, ndp=4, source_col=\"source\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9e0d138e-ab3e-488d-ab95-a36a0ac1ae8b", - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "if combine_mode != \"concatenate\":\n", - " if \"compared_to\" in df_final.columns.to_list():\n", - " vf.explain_intra_source_validation_output(\n", - " res,\n", - " top_k=5,\n", - " df_original=df_final_val, \n", - " ndp_used=4,\n", - " samples_per_source=2, \n", - " max_sources=5, \n", - " source_col=\"source\",\n", - " )" - ] - }, - { - "cell_type": "markdown", - "id": "220d3621-1cf4-4aa1-94f8-6a6f7dfb5985", - "metadata": {}, - "source": [ - "## Cross Catalog Deduplication Validation" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "57d2e1b6-0f02-4ff9-baab-a029cfce7315", - "metadata": {}, - "outputs": [], - "source": [ - "if combine_mode != \"concatenate\":\n", - " report = vf.validate_tie_results_fast(df_final_val, threshold=0.0005, max_groups=60000, include_rows=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "246692ac-5c4a-465c-8802-2ab6831b6e9a", - "metadata": {}, - "outputs": [], - "source": [ - "if combine_mode != \"concatenate\":\n", - " vf.explain_tie_validation_output(report, show_per_rule=3)" - ] - }, - { - "cell_type": "markdown", - "id": "3df7b3a3-d1f0-405e-b28a-b2c6b9a63e1d", - "metadata": {}, - "source": [ - "## Non-compared Validation" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "df988e3a-dda3-43fb-931a-415420204d70", - "metadata": {}, - "outputs": [], - "source": [ - "if combine_mode != \"concatenate\":\n", - " if \"compared_to\" in df_final.columns.to_list():\n", - " result = vf.render_na_compared_to_validation(df_final_val, show_max=10, assert_if_invalid=False)" - ] - }, - { - "cell_type": "markdown", - "id": "1754379f-3e1b-417c-bf27-8a36ce31ed26", - "metadata": {}, - "source": [ - "## Own Pipeline Product Validation" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0c9c3797-ca65-474e-9852-ec3ab2f7ecf6", - "metadata": {}, - "outputs": [], - "source": [ - "if combine_mode != \"concatenate\":\n", - " if \"compared_to\" in df_final.columns.to_list():\n", - " df_original = pd.read_parquet(\"../test_data/pipeline_generated_sample.parquet\")\n", - " df_processed = df_final[df_final[\"source\"] == \"019_pipeline_sample\"]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "38a98c9d-412d-44f8-bb1e-f720e3ab9146", - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "if combine_mode != \"concatenate\":\n", - " if \"compared_to\" in df_final.columns.to_list():\n", - " res = vf.validate_tie_preservation(df_original, df_processed, key=\"CRD_ID\")\n", - " print(vf.explain_tie_preservation(res))" - ] - }, - { - "cell_type": "markdown", - "id": "20f35e40-d969-4e3c-b91c-31bd567318ce", - "metadata": {}, - "source": [ - "## Manual Validation (Optional)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "fb72123e-ecb3-47b4-bac3-9e131207f02b", - "metadata": {}, - "outputs": [], - "source": [ - "if combine_mode != \"concatenate\":\n", - " if \"compared_to\" in df_final.columns.to_list():\n", - " df_for_manual = df_final[[\n", - " \"CRD_ID\",\"ra\",\"dec\",\"z\",\"survey\",\"source\",\n", - " \"tie_result\",\"compared_to\",\"z_flag_homogenized\",\n", - " \"instrument_type_homogenized\",\"group_id\"\n", - " ]]\n", - " else:\n", - " df_for_manual = df_final[[\n", - " \"CRD_ID\",\"ra\",\"dec\",\"z\",\"survey\",\"source\",\n", - " \"tie_result\",\"z_flag_homogenized\",\n", - " \"instrument_type_homogenized\",\"group_id\"\n", - " ]] \n", - " \n", - " results = vf.analyze_groups_by_group_id_fast(\n", - " df_for_manual,\n", - " threshold=0.0005,\n", - " max_groups=10000,\n", - " max_examples_per_case=5,\n", - " render=True,\n", - " compute_same_source_pair=True,\n", - " )" - ] - }, - { - "cell_type": "markdown", - "id": "18ce0a4d-c7cd-4e8e-966b-ea27d80e0b8b", - "metadata": {}, - "source": [ - "## Validation of groups with compared_to ``" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8b19b3c9-515c-4001-ac0b-63e021586552", - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "import numpy as np\n", - "\n", - "def check_group_id_integrity(\n", - " df: pd.DataFrame,\n", - " group_col: str = \"group_id\",\n", - " compared_col: str = \"compared_to\",\n", - " id_col: str = \"CRD_ID\",\n", - " zf_col: str = \"z_flag_homogenized\",\n", - " show_examples: int = 5,\n", - "):\n", - " if group_col not in df.columns:\n", - " raise KeyError(f\"'{group_col}' não está no DataFrame.\")\n", - "\n", - " # compared_to como string segura (ou vazio, se a coluna não existir)\n", - " if compared_col in df.columns:\n", - " cmp_str = df[compared_col].astype(\"string\").fillna(\"\").str.strip()\n", - " else:\n", - " cmp_str = pd.Series([\"\"] * len(df), index=df.index, dtype=\"string\")\n", - "\n", - " # flags linha-a-linha\n", - " cmp_nonempty = cmp_str.ne(\"\")\n", - " is_star = pd.Series(False, index=df.index)\n", - " if zf_col in df.columns:\n", - " is_star = pd.to_numeric(df[zf_col], errors=\"coerce\").eq(6)\n", - "\n", - " # agregações por group_id (inclui NaN se houver)\n", - " gkey = df[group_col]\n", - " size = gkey.groupby(gkey, dropna=False).size().rename(\"size\")\n", - " n_cmp_nonempty = cmp_nonempty.groupby(gkey, dropna=False).sum().rename(\"n_cmp_nonempty\")\n", - " n_star = is_star.groupby(gkey, dropna=False).sum().rename(\"n_star\")\n", - "\n", - " stats = pd.concat([size, n_cmp_nonempty, n_star], axis=1).fillna(0)\n", - " stats[\"n_cmp_nonempty\"] = stats[\"n_cmp_nonempty\"].astype(int)\n", - " stats[\"n_star\"] = stats[\"n_star\"].astype(int)\n", - "\n", - " # suspeitos = grupos com 2+ linhas e NENHUM compared_to preenchido\n", - " suspects = stats[(stats[\"size\"] >= 2) & (stats[\"n_cmp_nonempty\"] == 0)]\n", - "\n", - " print(f\"Grupos suspeitos (size ≥ 2 e compared_to vazio para todos): {len(suspects)}\")\n", - " if len(suspects) > 0:\n", - " display(suspects.sort_values([\"size\"], ascending=False).head(10))\n", - " ex_gids = suspects.index[:show_examples]\n", - " cols = [c for c in [id_col, \"survey\", \"source\", \"z\", zf_col, compared_col, group_col] if c in df.columns]\n", - " sample = df[df[group_col].isin(ex_gids)].loc[:, cols].sort_values([group_col, id_col], na_position=\"last\")\n", - " display(sample)\n", - "\n", - " # retorno útil para salvar/inspecionar depois\n", - " return {\"stats\": stats, \"suspects\": suspects}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d3f5890c-4b64-41e9-8a03-ffe9d3a8f369", - "metadata": {}, - "outputs": [], - "source": [ - "out = check_group_id_integrity(df_final)\n", - "assert len(out[\"suspects\"]) == 0, \"There is group_id with size>=2 and compared_to empty!\"" - ] - }, - { - "cell_type": "markdown", - "id": "801271a1-eeb4-402d-a97c-46de5cb47376", - "metadata": {}, - "source": [ - "## Validation - Prepared Catalogs" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "54c38638-040b-43cc-9c0b-fb22ba0e0dd8", - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "# Dicionário de regras equivalente ao YAML\n", - "translation_rules = {\n", - " \"2DFGRS\": {\n", - " \"z_flag_translation\": {1: 0, 2: 1, 3: 3, 4: 4, 5: 4},\n", - " \"instrument_type_translation\": {\"default\": \"s\"},\n", - " },\n", - " \"2DFLENS\": {\n", - " \"z_flag_translation\": {1: 0, 2: 1, 3: 3, 4: 4, 6: 6},\n", - " \"instrument_type_translation\": {\"default\": \"s\"},\n", - " },\n", - " \"2MRS\": {\n", - " \"z_flag_translation\": {\n", - " \"conditions\": [\n", - " {\"expr\": \"z_err == 0\", \"value\": 3},\n", - " {\"expr\": \"0 < z_err < 0.0005\", \"value\": 4},\n", - " {\"expr\": \"z_err >= 0.0005\", \"value\": 3},\n", - " ],\n", - " \"default\": 0,\n", - " },\n", - " \"instrument_type_translation\": {\"default\": \"s\"},\n", - " },\n", - " \"3D-HST\": {\n", - " \"z_flag_translation\": {\n", - " \"conditions\": [\n", - " {\"expr\": \"z_best_s == 0\", \"value\": 6},\n", - " {\"expr\": \"z_best_s == 1 and z_spec != -1\", \"value\": 4},\n", - " {\"expr\": \"z_best_s == 2 and use_zgrism == 1 and flag1 == 0 and flag2 == 0\", \"value\": 3},\n", - " {\"expr\": \"z_best_s == 3 and use_phot == 1\", \"value\": 3},\n", - " ],\n", - " \"default\": 0,\n", - " },\n", - " \"instrument_type_translation\": {\n", - " \"conditions\": [\n", - " {\"expr\": \"z_best_s == 1\", \"value\": \"s\"},\n", - " {\"expr\": \"z_best_s == 2\", \"value\": \"g\"},\n", - " {\"expr\": \"z_best_s == 3\", \"value\": \"p\"},\n", - " ],\n", - " \"default\": \"g\",\n", - " },\n", - " },\n", - " \"6DFGS\": {\n", - " \"z_flag_translation\": {1: 0, 2: 1, 3: 3, 4: 4, 6: 6},\n", - " \"instrument_type_translation\": {\"default\": \"s\"},\n", - " },\n", - " \"ASTRODEEP\": {\n", - " \"z_flag_translation\": {\n", - " \"conditions\": [\n", - " {\"expr\": \"zspec_survey != '-'\", \"value\": 4},\n", - " {\"expr\": \"zspec_survey == '-'\", \"value\": 3},\n", - " ],\n", - " \"default\": 0,\n", - " },\n", - " \"instrument_type_translation\": {\n", - " \"conditions\": [\n", - " {\"expr\": \"zspec_survey != '-'\", \"value\": \"s\"},\n", - " {\"expr\": \"zspec_survey == '-'\", \"value\": \"p\"},\n", - " ],\n", - " \"default\": \"p\",\n", - " },\n", - " },\n", - " \"ASTRODEEP-JWST\": {\n", - " \"z_flag_translation\": {\n", - " \"conditions\": [\n", - " {\"expr\": \"zspec != -99 and z_flag < 400 and (len(str(int(z_flag))) <= 1 or int(str(int(z_flag))[-2]) <= 3)\", \"value\": 4},\n", - " {\"expr\": \"zspec == -99 and z_flag < 400 and (len(str(int(z_flag))) <= 1 or int(str(int(z_flag))[-2]) <= 3)\", \"value\": 3},\n", - " ],\n", - " \"default\": 0,\n", - " },\n", - " \"instrument_type_translation\": {\n", - " \"conditions\": [\n", - " {\"expr\": \"zspec != -99\", \"value\": \"s\"},\n", - " {\"expr\": \"zspec == -99\", \"value\": \"p\"},\n", - " ],\n", - " \"default\": \"p\",\n", - " },\n", - " },\n", - " \"DESI\": {\n", - " \"z_flag_translation\": {\n", - " \"conditions\": [\n", - " {\"expr\": \"ZCAT_PRIMARY != True\", \"value\": 0},\n", - " {\"expr\": \"z_flag != 0 and ZCAT_PRIMARY == True\", \"value\": 1},\n", - " {\"expr\": \"z_flag == 0 and ZCAT_PRIMARY == True and z_err < 0.0005\", \"value\": 4},\n", - " {\"expr\": \"z_flag == 0 and ZCAT_PRIMARY == True and z_err >= 0.0005\", \"value\": 3},\n", - " ],\n", - " \"default\": 0,\n", - " },\n", - " \"instrument_type_translation\": {\"default\": \"s\"},\n", - " },\n", - " \"JADES\": {\n", - " \"z_flag_translation\": {4: 4, 3: 3, 2: 2, 1: 1, 0: 0},\n", - " \"instrument_type_translation\": {\"default\": \"s\"},\n", - " },\n", - " \"MOSDEF\": {\n", - " \"z_flag_translation\": {7: 4, 6: 3, 5: 2, 4: 2, 3: 1, 2: 1, 1: 0, 0: 0},\n", - " \"instrument_type_translation\": {\"default\": \"s\"},\n", - " },\n", - " \"OZDES\": {\n", - " \"z_flag_translation\": {1: 0, 2: 1, 3: 3, 4: 4, 6: 6},\n", - " \"instrument_type_translation\": {\"default\": \"s\"},\n", - " },\n", - " \"PRIMUS\": {\n", - " \"z_flag_translation\": {-1: 0, 2: 1, 3: 2, 4: 3},\n", - " \"instrument_type_translation\": {\"default\": \"g\"},\n", - " },\n", - " \"VANDELS\": {\n", - " \"z_flag_translation\": {\n", - " 0: 0, 1: 1, 2: 2, 3: 4, 4: 4, 9: 3,\n", - " 10: 0, 11: 1, 12: 2, 13: 4, 14: 4, 19: 3,\n", - " 20: 0, 21: 1, 22: 2, 23: 4, 24: 4, 29: 3,\n", - " 210: 0, 211: 1, 212: 2, 213: 4, 214: 4, 219: 3,\n", - " },\n", - " \"instrument_type_translation\": {\"default\": \"s\"},\n", - " },\n", - " \"VIMOS\": {\n", - " \"z_flag_translation\": {4: 4, 3: 3, 2: 2, 1: 1, 0: 0},\n", - " \"instrument_type_translation\": {\"default\": \"s\"},\n", - " },\n", - " \"VUDS\": {\n", - " \"z_flag_translation\": {\n", - " 1: 1, 11: 1, 21: 1, 31: 1, 41: 1,\n", - " 2: 2, 12: 2, 22: 2, 32: 2, 42: 2, 9: 2, 19: 2, 29: 2, 39: 2, 49: 2,\n", - " 3: 3, 13: 3, 23: 3, 33: 3, 43: 3,\n", - " 4: 4, 14: 4, 24: 4, 34: 4, 44: 4,\n", - " },\n", - " \"instrument_type_translation\": {\"default\": \"s\"},\n", - " },\n", - " \"VVDS\": {\n", - " \"z_flag_translation\": {\n", - " 0: 0, 1: 1, 2: 2, 3: 3, 4: 4, 9: 2,\n", - " 10: 0, 11: 1, 12: 2, 13: 3, 14: 4, 19: 2,\n", - " 20: 0, 21: 1, 22: 2, 23: 3, 24: 4, 29: 2,\n", - " 210: 0, 211: 1, 212: 2, 213: 3, 214: 4, 219: 2,\n", - " },\n", - " \"instrument_type_translation\": {\"default\": \"s\"},\n", - " },\n", - "\n", - " # Special cases using continuous rule and inherited type\n", - " \"CANDELS\": {\"_special\": \"CANDELS_NED\"},\n", - " \"NED\": {\"_special\": \"CANDELS_NED\"},\n", - "}\n", - "\n", - "def _safe_eval_expr(expr: str, ctx: dict) -> bool:\n", - " \"\"\"\n", - " Avalia 'expr' usando apenas variáveis do ctx e funções básicas.\n", - " Retorna True/False; se der erro, retorna False.\n", - " \"\"\"\n", - " try:\n", - " # Permitir apenas funções básicas e numpy\n", - " allowed_globals = {\n", - " \"__builtins__\": {\"len\": len, \"int\": int, \"str\": str, \"float\": float},\n", - " \"np\": np,\n", - " }\n", - " return bool(eval(expr, allowed_globals, ctx))\n", - " except Exception:\n", - " return False\n", - "\n", - "def _apply_translation(value_map, row_ctx):\n", - " \"\"\"\n", - " value_map pode ser:\n", - " - dict simples {orig: dest} (pode conter 'default')\n", - " - dict com 'conditions' (lista de {expr, value}) e opcional 'default'\n", - " Retorna (valor_traduzido, matched_bool)\n", - " \"\"\"\n", - " if isinstance(value_map, dict) and \"conditions\" in value_map:\n", - " for cond in value_map[\"conditions\"]:\n", - " expr = cond.get(\"expr\", \"\")\n", - " val = cond.get(\"value\", np.nan)\n", - " if expr and _safe_eval_expr(expr, row_ctx):\n", - " return val, True\n", - " # nenhum matched -> usa default se houver\n", - " if \"default\" in value_map:\n", - " return value_map[\"default\"], True\n", - " return np.nan, False\n", - "\n", - " # mapeamento direto (sem 'conditions'):\n", - " if isinstance(value_map, dict):\n", - " key = row_ctx.get(\"z_flag\", np.nan)\n", - " if key in value_map:\n", - " return value_map[key], True\n", - " # Se não houver chave correspondente, mas existir 'default', use-o\n", - " if \"default\" in value_map:\n", - " return value_map[\"default\"], True\n", - " return np.nan, False\n", - "\n", - " return np.nan, False\n", - "\n", - "def validate_row(row):\n", - " survey = row.get(\"survey\", None)\n", - "\n", - " # construir contexto com None -> np.nan, para evitar erros de comparação\n", - " ctx = {}\n", - " for k, v in row.items():\n", - " ctx[k] = (np.nan if v is None else v)\n", - "\n", - " # Casos especiais (CANDELS e NED): regra contínua 0..1 e type herdado\n", - " if survey in (\"CANDELS\", \"NED\"):\n", - " x = row.get(\"z_flag\", np.nan)\n", - " # z_flag esperado:\n", - " if x == 0.0:\n", - " z_expected = 0.0\n", - " elif (isinstance(x, (float, int))) and (0.0 < x < 0.7):\n", - " z_expected = 1.0\n", - " elif (isinstance(x, (float, int))) and (0.7 <= x < 0.9):\n", - " z_expected = 2.0\n", - " elif (isinstance(x, (float, int))) and (0.9 <= x < 0.99):\n", - " z_expected = 3.0\n", - " elif (isinstance(x, (float, int))) and (0.99 <= x <= 1.0):\n", - " z_expected = 4.0\n", - " else:\n", - " z_expected = np.nan\n", - "\n", - " # type_expected é o próprio 'type' da linha\n", - " type_expected = row.get(\"instrument_type\", np.nan)\n", - " return z_expected, type_expected\n", - "\n", - " # Regras gerais dos surveys\n", - " rules = translation_rules.get(survey, None)\n", - " if rules is None:\n", - " return np.nan, np.nan\n", - "\n", - " # z_flag_homogenized esperado\n", - " z_rules = rules.get(\"z_flag_translation\", None)\n", - " if z_rules is None:\n", - " z_expected = np.nan\n", - " else:\n", - " z_expected, _ = _apply_translation(z_rules, ctx)\n", - "\n", - " # instrument_type_homogenized esperado\n", - " t_rules = rules.get(\"instrument_type_translation\", None)\n", - " if t_rules is None:\n", - " type_expected = np.nan\n", - " else:\n", - " if isinstance(t_rules, dict) and (\"conditions\" in t_rules or \"default\" in t_rules):\n", - " type_expected, matched = _apply_translation(t_rules, ctx)\n", - " else:\n", - " type_expected, matched = _apply_translation(t_rules, ctx)\n", - "\n", - "\n", - " return z_expected, type_expected\n", - "\n", - "\n", - "# =========================================================\n", - "# VALIDATE TRANSLATIONS IN TEMP FILES\n", - "# =========================================================\n", - "merged_files = glob.glob(os.path.join(prepared_temp_dir, \"prepared*/*.parquet\"))\n", - "merged_files = [f for f in merged_files if \"pipeline_sample\" not in f]\n", - "\n", - "if not merged_files:\n", - " print(\"⚠️ No prepared parquet files found for validation.\")\n", - "else:\n", - " issues = []\n", - " \n", - " for merged_file in merged_files:\n", - " print(f\"🔍 Validating {merged_file}\")\n", - " df = pd.read_parquet(merged_file)\n", - " \n", - " for _, row in df.iterrows():\n", - " z_exp, type_exp = validate_row(row)\n", - " \n", - " if not (pd.isna(z_exp) and pd.isna(row[\"z_flag_homogenized\"])) and z_exp != row[\"z_flag_homogenized\"]:\n", - " issue = row.to_dict()\n", - " issue[\"field\"] = \"z_flag_homogenized\"\n", - " issue[\"expected\"] = z_exp\n", - " issue[\"found\"] = row[\"z_flag_homogenized\"]\n", - " issues.append(issue)\n", - " \n", - " if not (pd.isna(type_exp) and pd.isna(row[\"instrument_type_homogenized\"])) and type_exp != row[\"instrument_type_homogenized\"]:\n", - " issue = row.to_dict()\n", - " issue[\"field\"] = \"instrument_type_homogenized\"\n", - " issue[\"expected\"] = type_exp\n", - " issue[\"found\"] = row[\"instrument_type_homogenized\"]\n", - " issues.append(issue)\n", - " \n", - " if issues:\n", - " issues_df = pd.DataFrame(issues)\n", - " display(issues_df)\n", - " print(f\"⚠️ {len(issues)} mismatches found!\")\n", - " else:\n", - " print(\"✅ All homogenized fields match the expected values.\")" - ] - }, - { - "cell_type": "markdown", - "id": "ede0adcb-e34d-4e81-90e1-8079773eedb6", - "metadata": {}, - "source": [ - "# Time Profiler" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "417dc417-cee6-45ba-8306-952ac9bf03f4", - "metadata": {}, - "outputs": [], - "source": [ - "import os, re\n", - "from datetime import datetime, timedelta\n", - "import matplotlib.pyplot as plt\n", - "from collections import defaultdict\n", - "\n", - "log_path = \"process001/process_info/pipeline.log\"\n", - "INIT_LEFT_PAD_S = 3\n", - "\n", - "# ---------- Regex ----------\n", - "TS = r\"(?P\\d{4}-\\d{2}-\\d{2}-\\d{2}:\\d{2}:\\d{2}\\.\\d+)\"\n", - "PH = r\"(?PSTART|END)\"\n", - "\n", - "# Fases macro da coluna \"stage\" (3ª coluna): preparation, automatch, crossmatch, deduplication, consolidation, register\n", - "PHASES = r\"(?:preparation|automatch|crossmatch|deduplication|consolidation|register)\"\n", - "RE_PHASE = re.compile(\n", - " fr\"{TS}\\s*\\|\\s*INFO\\s*\\|\\s*(?P{PHASES})\\s*\\|\\s*crc\\s*\\|\\s*{PH}\\s+(?P.*)$\"\n", - ")\n", - "\n", - "# init (bootstrap)\n", - "RE_INIT = re.compile(fr\"{TS}\\s*\\|\\s*INFO\\s*\\|\\s*init\\s*\\|\\s*crc\\s*\\|\\s*{PH}\\s+init:\", re.X)\n", - "\n", - "# prepare_catalog (produto)\n", - "RE_PREPARE = re.compile(\n", - " fr\"{TS}\\s*\\|\\s*INFO\\s*\\|\\s*preparation\\s*\\|\\s*crc\\.specz\\s*\\|\\s*{PH}\\s+prepare_catalog\\s+product=(?P[\\w.\\-\\d]+)\",\n", - " re.X,\n", - ")\n", - "\n", - "# automatch (artifact)\n", - "RE_AUTOMATCH = re.compile(\n", - " fr\"{TS}\\s*\\|\\s*INFO\\s*\\|\\s*automatch\\s*\\|\\s*crc\\.crossmatch_auto\\s*\\|\\s*{PH}\\s+automatch:\\s+artifact=(?P[\\w\\.\\-\\d]+)\",\n", - " re.X,\n", - ")\n", - "\n", - "# crossmatch (step)\n", - "RE_XMATCH = re.compile(\n", - " fr\"{TS}\\s*\\|\\s*INFO\\s*\\|\\s*crossmatch\\s*\\|\\s*crc\\.crossmatch\\s*\\|\\s*{PH}\\s+crossmatch_update_compared_to:\\s+step=(?P\\d+)\",\n", - " re.X,\n", - ")\n", - "\n", - "def parse_ts(s: str) -> datetime:\n", - " return datetime.strptime(s, \"%Y-%m-%d-%H:%M:%S.%f\")\n", - "\n", - "# ---------- Varredura ----------\n", - "start_times, end_times = {}, {}\n", - "\n", - "def reg(task_type: str, ident: str, ts: datetime, ph: str):\n", - " key = f\"{task_type}|{ident}\"\n", - " if ph == \"START\":\n", - " if key not in start_times or ts < start_times[key]:\n", - " start_times[key] = ts\n", - " else:\n", - " if key not in end_times or ts > end_times[key]:\n", - " end_times[key] = ts\n", - "\n", - "if not os.path.exists(log_path):\n", - " raise FileNotFoundError(log_path)\n", - "\n", - "with open(log_path, \"r\", encoding=\"utf-8\", errors=\"ignore\") as f:\n", - " for line in f:\n", - " # INIT (específico)\n", - " m = RE_INIT.search(line)\n", - " if m:\n", - " ts = parse_ts(m.group(\"ts\")); ph = m.group(\"ph\")\n", - " reg(\"init\", \"init\", ts, ph)\n", - " continue\n", - "\n", - " # PREPARE por produto (específico)\n", - " m = RE_PREPARE.search(line)\n", - " if m:\n", - " ts = parse_ts(m.group(\"ts\")); ph = m.group(\"ph\")\n", - " reg(\"prepare_catalog\", m.group(\"name\"), ts, ph)\n", - " continue\n", - "\n", - " # AUTOMATCH por artifact (específico)\n", - " m = RE_AUTOMATCH.search(line)\n", - " if m:\n", - " ts = parse_ts(m.group(\"ts\")); ph = m.group(\"ph\")\n", - " reg(\"automatch\", m.group(\"name\"), ts, ph)\n", - " continue\n", - "\n", - " # CROSSMATCH step=N (específico)\n", - " m = RE_XMATCH.search(line)\n", - " if m:\n", - " ts = parse_ts(m.group(\"ts\")); ph = m.group(\"ph\")\n", - " reg(\"crossmatch\", f\"step{int(m.group('step')):02d}\", ts, ph)\n", - " continue\n", - "\n", - " # Fase macro (genérica; por último!)\n", - " m = RE_PHASE.search(line)\n", - " if m:\n", - " ts = parse_ts(m.group(\"ts\")); ph = m.group(\"ph\")\n", - " phase = m.group(\"phase\")\n", - " reg(f\"{phase}_phase\", phase, ts, ph)\n", - " continue\n", - "\n", - "# ---------- Filtra completas ----------\n", - "all_keys = [k for k in start_times if k in end_times]\n", - "if not all_keys:\n", - " raise RuntimeError(\"Nenhuma tarefa com START e END encontrada no log.\")\n", - "\n", - "# estética\n", - "if \"init|init\" in start_times:\n", - " start_times[\"init|init\"] -= timedelta(seconds=INIT_LEFT_PAD_S)\n", - "\n", - "# ---------- Ordenação ----------\n", - "def group_of(key: str) -> str:\n", - " return key.split(\"|\", 1)[0]\n", - "\n", - "def start_of(key: str) -> datetime:\n", - " return start_times[key]\n", - "\n", - "def crossmatch_sort_key(key: str):\n", - " s = key.split(\"|\", 1)[1].replace(\"step\", \"\")\n", - " return int(s) if s.isdigit() else 10**9\n", - "\n", - "groups = [\n", - " \"init\",\n", - " \"preparation_phase\", # barra macro da fase\n", - " \"prepare_catalog\", # barras por produto\n", - " \"automatch_phase\",\n", - " \"automatch\", # barras por artifact\n", - " \"crossmatch_phase\",\n", - " \"crossmatch\", # barras por step\n", - " \"deduplication_phase\",\n", - " \"consolidation_phase\",\n", - " \"register_phase\", # aparece só se existir no log\n", - "]\n", - "\n", - "\n", - "palette = {\n", - " \"init\": \"#1f77b4\",\n", - " \"preparation_phase\": \"#ff7f0e\",\n", - " \"prepare_catalog\": \"#ffbb78\",\n", - " \"automatch_phase\": \"#2ca02c\",\n", - " \"automatch\": \"#98df8a\",\n", - " \"crossmatch_phase\": \"#d62728\",\n", - " \"crossmatch\": \"#ff9896\",\n", - " \"deduplication_phase\": \"#9467bd\",\n", - " \"consolidation_phase\": \"#8c564b\",\n", - " \"register_phase\": \"#e377c2\",\n", - "}\n", - "\n", - "\n", - "ordered_keys = []\n", - "for g in groups:\n", - " gkeys = [k for k in all_keys if group_of(k) == g]\n", - " if not gkeys:\n", - " continue\n", - " if g == \"crossmatch\":\n", - " gkeys = sorted(gkeys, key=crossmatch_sort_key)\n", - " else:\n", - " gkeys = sorted(gkeys, key=start_of)\n", - " ordered_keys.extend(gkeys)\n", - "\n", - "# --- Injeta register_phase se não existir ---\n", - "if not any(k.startswith(\"register_phase|\") for k in start_times):\n", - " if ordered_keys:\n", - " last_end = max(end_times[k] for k in ordered_keys)\n", - " else:\n", - " last_end = min(start_times.values())\n", - " reg_start = last_end + timedelta(seconds=0.5)\n", - " reg_end = reg_start + timedelta(seconds=1.0)\n", - " start_times[\"register_phase|register\"] = reg_start\n", - " end_times[\"register_phase|register\"] = reg_end\n", - " ordered_keys.append(\"register_phase|register\")\n", - "\n", - "# ---------- Referência temporal ----------\n", - "t0 = min(start_times[k] for k in ordered_keys)\n", - "starts = [(start_times[k] - t0).total_seconds() for k in ordered_keys]\n", - "ends = [(end_times[k] - t0).total_seconds() for k in ordered_keys]\n", - "\n", - "# ---------- Plot ----------\n", - "fig, ax = plt.subplots(figsize=(15, 8))\n", - "\n", - "# pular os globais destas três fases\n", - "skip_globals = {\"preparation_phase\", \"automatch_phase\", \"crossmatch_phase\"}\n", - "\n", - "# desenha as barras (apenas uma vez)\n", - "for i, k in enumerate(ordered_keys):\n", - " g = group_of(k)\n", - " if g in skip_globals:\n", - " continue # não desenha as barras globais dessas três fases\n", - "\n", - " c = palette.get(g, \"#444444\")\n", - " lw = 4 if g.endswith(\"_phase\") else 2 # fases globais restantes (dedup/consolid/register) ficam mais grossas\n", - " ax.hlines(y=i, xmin=starts[i], xmax=ends[i], colors=c, linewidth=lw)\n", - " ax.scatter([starts[i], ends[i]], [i, i], s=14, color=c, zorder=3)\n", - "\n", - "# tira os labels default do eixo Y\n", - "ax.set_yticks(range(len(ordered_keys)))\n", - "ax.set_yticklabels([\"\"] * len(ordered_keys))\n", - "\n", - "# adiciona só um label por bloco, no centro\n", - "def add_block_label(prefix, text):\n", - " idxs = [i for i, k in enumerate(ordered_keys) if k.startswith(prefix)]\n", - " if idxs:\n", - " mid = (min(idxs) + max(idxs)) / 2\n", - " ax.text(-5, mid, text, va=\"center\", ha=\"right\", fontsize=10, fontweight=\"bold\")\n", - "\n", - "add_block_label(\"init\", \"init\")\n", - "add_block_label(\"prepare_catalog\", \"preparation\")\n", - "add_block_label(\"automatch\", \"automatch\")\n", - "add_block_label(\"crossmatch\", \"crossmatch\")\n", - "add_block_label(\"deduplication_phase\", \"deduplication\")\n", - "add_block_label(\"consolidation_phase\", \"consolidation\")\n", - "add_block_label(\"register_phase\", \"register\")\n", - "\n", - "\n", - "# eixo X e margens (inclui espaço à esquerda para os rótulos em x=-5)\n", - "xmax = max(ends) if ends else 1.0\n", - "ax.set_xlim(-6, xmax * 1.02)\n", - "ax.set_xlabel(\"Time (s)\", fontsize=12)\n", - "ax.grid(True, linestyle=\"--\", alpha=0.3, axis=\"x\")\n", - "ax.set_title(\"CRC – Time Profile (pipeline.log)\", fontsize=16)\n", - "\n", - "# dá espaço para os rótulos à esquerda sem depender do tight_layout\n", - "plt.subplots_adjust(left=0.20, right=0.98, top=0.92, bottom=0.08)\n", - "plt.show()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "pipe_crd", - "language": "python", - "name": "pipe_crd" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.13.11" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks/validation_functions.py b/notebooks/validation_functions.py deleted file mode 100644 index 40625c0..0000000 --- a/notebooks/validation_functions.py +++ /dev/null @@ -1,2003 +0,0 @@ -# validation_functions.py -from __future__ import annotations - -# ============================== -# Standard library -# ============================== -import math -from collections import defaultdict -from itertools import combinations -from typing import Dict, List, Tuple - -# ============================== -# Third-party libraries -# ============================== -import numpy as np -import pandas as pd -from scipy.sparse import coo_matrix -from scipy.sparse.csgraph import connected_components -from IPython.display import display, Markdown - - -def _norm_str(s): - """Normalize strings; NA-like -> None.""" - if s is None: - return None - s = str(s).strip() - if not s or s.lower() in {"na", "nan", "", "none", "null"}: - return None - return s - -# ======================= -# ======================= -# Intra Source -# ======================= -# ======================= -def _parse_cmp_list_fast(s: str) -> list[str]: - """Parse a comma-separated list into trimmed items; NA-like -> [].""" - if s is None: - return [] - s = str(s).strip() - if not s or s.lower() in {"na", "nan", "", "none", "null"}: - return [] - return [x.strip() for x in s.split(",") if x.strip()] - -def validate_intra_source_cells_fast( - df_final: pd.DataFrame, - ndp: int = 4, - source_col: str = "source", - emit_pairs: bool = True, - limit_pairs: int | None = None, -): - """Fast intra-source validator on (source, ra4, dec4) cells. - - Builds directed edges inside each cell using explode and counts pair directions. - - Args: - df_final: Input dataframe. - ndp: Decimals to round RA/DEC. - source_col: Source/catalog column. - emit_pairs: Whether to emit the pairs dataframe. - limit_pairs: Optional cap to avoid materializing all missing pairs. - - Returns: - Dict with pairs, cell_summary, totals, violations, diag_sources. - """ - # Minimal normalization (avoid unnecessary copies) - df = df_final[[source_col, "CRD_ID", "ra", "dec", "compared_to"]].copy() - df["CRD_ID"] = df["CRD_ID"].astype(str) - # normalize source - df[source_col] = df[source_col].map(_norm_str).astype("string") - # rounded cell - df["ra4"] = pd.to_numeric(df["ra"], errors="coerce").round(ndp) - df["dec4"] = pd.to_numeric(df["dec"], errors="coerce").round(ndp) - - # Keep only cells with 2+ objects - cell_sizes = ( - df.groupby([source_col, "ra4", "dec4"], dropna=False) - .size().rename("n").reset_index() - ) - multi_cells = cell_sizes[cell_sizes["n"] >= 2] - if multi_cells.empty: - cell_summary = pd.DataFrame(columns=["source","ra4","dec4","n_pairs","n_ok_bi","n_ok_any","n_missing","bi_cov","any_cov"]) - totals = {"n_cells": 0, "n_pairs": 0, "n_ok_bi": 0, "n_ok_any": 0, "n_missing": 0} - return { - "pairs": pd.DataFrame(columns=["source","ra4","dec4","A","B","A_in_B","B_in_A","status"]), - "cell_summary": cell_summary, - "totals": totals, - "violations": pd.DataFrame(columns=["source","ra4","dec4","A","B","A_in_B","B_in_A","status"]), - "diag_sources": pd.DataFrame(columns=["source","n_cells","cells_2plus"]), - } - - # Subset with 2+ cells only - df2 = df.merge(multi_cells[[source_col, "ra4", "dec4"]].drop_duplicates(), - on=[source_col, "ra4", "dec4"], how="inner") - - # Explode compared_to -> directed edges inside the cell - df2["_cmp_list"] = df2["compared_to"].map(_parse_cmp_list_fast) - edges = ( - df2[[source_col, "ra4", "dec4", "CRD_ID", "_cmp_list"]] - .explode("_cmp_list", ignore_index=True) - .rename(columns={"CRD_ID": "A", "_cmp_list": "B"}) - ) - edges = edges[edges["B"].notna() & (edges["B"] != "")] - # Keep only edges whose B also belongs to the same cell - members = df2[[source_col, "ra4", "dec4", "CRD_ID"]].rename(columns={"CRD_ID":"B"}) - edges = edges.merge(members, on=[source_col, "ra4", "dec4", "B"], how="inner") - - if edges.empty: - # No directed edges -> all pairs are "missing". - # Compute aggregates only (fast). - n_pairs_series = (multi_cells["n"] * (multi_cells["n"] - 1) // 2) - cell_summary = multi_cells.assign( - n_pairs=n_pairs_series, - n_ok_bi=0, n_ok_any=0, n_missing=n_pairs_series, - bi_cov=0.0, any_cov=0.0 - ).rename(columns={source_col:"source"}).drop(columns=["n"]) - totals = { - "n_cells": int(cell_summary.shape[0]), - "n_pairs": int(cell_summary["n_pairs"].sum()), - "n_ok_bi": 0, - "n_ok_any": 0, - "n_missing": int(cell_summary["n_pairs"].sum()), - } - diag_sources = ( - multi_cells.groupby(source_col)["n"] - .agg(n_cells="size", cells_2plus=lambda s: (s >= 2).sum()) - .reset_index().rename(columns={source_col:"source"}) - .sort_values("cells_2plus", ascending=False) - ) - return { - "pairs": pd.DataFrame(columns=["source","ra4","dec4","A","B","A_in_B","B_in_A","status"]), - "cell_summary": cell_summary, - "totals": totals, - "violations": pd.DataFrame(columns=["source","ra4","dec4","A","B","A_in_B","B_in_A","status"]), - "diag_sources": diag_sources, - } - - # Collapse to undirected pairs and count directions (1=one_way, 2=bi) - a_le_b = (edges["A"] <= edges["B"]) - edges["A_und"] = np.where(a_le_b, edges["A"], edges["B"]) - edges["B_und"] = np.where(a_le_b, edges["B"], edges["A"]) - - pair_counts = ( - edges.groupby([source_col, "ra4", "dec4", "A_und", "B_und"], dropna=False) - .size().rename("n_dir").reset_index() - ) - pair_counts["status"] = np.where(pair_counts["n_dir"] >= 2, "ok_bi", "ok_one_way") - - # Per-cell aggregates (fast) - per_cell_found = ( - pair_counts.groupby([source_col, "ra4", "dec4"], dropna=False) - .agg(n_ok_bi=("status", lambda s: (s == "ok_bi").sum()), - n_ok_any=("status", "size")) - .reset_index() - ) - cell_summary = multi_cells.merge(per_cell_found, on=[source_col, "ra4", "dec4"], how="left").fillna({"n_ok_bi":0, "n_ok_any":0}) - cell_summary["n_pairs"] = (cell_summary["n"] * (cell_summary["n"] - 1) // 2) - cell_summary["n_missing"] = cell_summary["n_pairs"] - cell_summary["n_ok_any"] - cell_summary["bi_cov"] = np.where(cell_summary["n_pairs"]>0, cell_summary["n_ok_bi"]/cell_summary["n_pairs"], 0.0) - cell_summary["any_cov"] = np.where(cell_summary["n_pairs"]>0, cell_summary["n_ok_any"]/cell_summary["n_pairs"], 0.0) - cell_summary = (cell_summary - .rename(columns={source_col:"source"}) - .drop(columns=["n"]) - .reset_index(drop=True)) - - totals = { - "n_cells": int(cell_summary.shape[0]), - "n_pairs": int(cell_summary["n_pairs"].sum()), - "n_ok_bi": int((pair_counts["status"] == "ok_bi").sum()), - "n_ok_any": int(pair_counts.shape[0]), - "n_missing": int(cell_summary["n_missing"].sum()), - } - - # Optional pair emission (include "missing" only if requested) - if not emit_pairs: - pairs = pd.DataFrame(columns=["source","ra4","dec4","A","B","A_in_B","B_in_A","status"]) - violations = pairs.copy() - else: - # Found pairs (fast) - pairs_found = pair_counts.rename(columns={"A_und":"A", "B_und":"B"}) - # Rebuild direction flags via join on directed edges - dir1 = edges[["A","B",source_col,"ra4","dec4"]].assign(dir=1) - dir2 = edges.rename(columns={"A":"B","B":"A"})[["A","B",source_col,"ra4","dec4"]].assign(dir=2) - both_dir = pd.concat([dir1, dir2], ignore_index=True) - mark = both_dir.drop_duplicates([source_col,"ra4","dec4","A","B"]).assign(val=True) - pairs_found = pairs_found.merge( - mark.rename(columns={"A":"A","B":"B","val":"A_in_B"})[[source_col,"ra4","dec4","A","B","A_in_B"]], - on=[source_col,"ra4","dec4","A","B"], how="left" - ) - pairs_found = pairs_found.merge( - mark.rename(columns={"A":"B","B":"A","val":"B_in_A"})[[source_col,"ra4","dec4","A","B","B_in_A"]], - on=[source_col,"ra4","dec4","A","B"], how="left" - ) - pairs_found["A_in_B"] = pairs_found["A_in_B"].fillna(False) - pairs_found["B_in_A"] = pairs_found["B_in_A"].fillna(False) - - pairs_found = (pairs_found - .rename(columns={source_col:"source"}) - [["source","ra4","dec4","A","B","A_in_B","B_in_A","status"]] - .reset_index(drop=True)) - - # If "missing" are needed, materialize per cell in a controlled way - missing_rows = [] - if limit_pairs is None or pairs_found.shape[0] < limit_pairs: - found_key = set( - (r.source, r.ra4, r.dec4, r.A, r.B) - for r in pairs_found.itertuples(index=False) - ) - for rec in multi_cells.itertuples(index=False): - src, ra4, dec4, n = getattr(rec, source_col), rec.ra4, rec.dec4, rec.n - ids = df2[(df2[source_col]==src) & (df2["ra4"].eq(ra4)) & (df2["dec4"].eq(dec4))]["CRD_ID"].tolist() - ids.sort() - for a, b in combinations(ids, 2): - key = (src, ra4, dec4, a, b) - if key not in found_key: - missing_rows.append({"source":src, "ra4":ra4, "dec4":dec4, - "A":a, "B":b, "A_in_B":False, "B_in_A":False, "status":"missing"}) - missing_df = pd.DataFrame(missing_rows) if missing_rows else pd.DataFrame(columns=pairs_found.columns) - pairs = pd.concat([pairs_found, missing_df], ignore_index=True) - else: - pairs = pairs_found - - violations = pairs[pairs["status"] != "ok_bi"].copy() - - # Diagnostics by source - diag_sources = ( - multi_cells.groupby(source_col)["n"] - .agg(n_cells="size", cells_2plus=lambda s: (s >= 2).sum()) - .reset_index().rename(columns={source_col:"source"}) - .sort_values("cells_2plus", ascending=False) - ) - - return { - "pairs": pairs, - "cell_summary": cell_summary, - "totals": totals, - "violations": violations, - "diag_sources": diag_sources, - } - -def explain_intra_source_validation_output( - res, - top_k: int = 20, - df_original: pd.DataFrame | None = None, - ndp_used: int = 4, - samples_per_source: int = 2, - max_sources: int | None = 5, - source_col: str = "source", -): - """Render explanations and samples for the intra-source validator.""" - # --- TOTALS --- - tot = res.get("totals", {}) - display(Markdown( -f""" -### Totals (`res["totals"]`) -- **`n_cells`**: number of cells (**`{source_col}` + `ra`/`dec` rounded to {ndp_used} decimals**) with **≥ 2 objects**. -- **`n_pairs`**: total number of A–B pairs evaluated in those cells. -- **`n_ok_bi`**: pairs with **bidirectionality** (A∈`compared_to`(B) **and** B∈`compared_to`(A)). -- **`n_ok_any`**: pairs with **at least one** direction present (A∈B **or** B∈A). -- **`n_missing`**: pairs with **no direction** present. - -**Values:** `{tot}` -""" - )) - - # === SAMPLES BY SOURCE (right after Totals) === - if df_original is not None: - display(Markdown( -f""" -### Sample groups by `{source_col}` -Below, for each `{source_col}` with at least one cell containing ≥2 objects (at {ndp_used} decimals), -we show up to **{samples_per_source}** cells (the largest ones) and list their objects with: - -`CRD_ID, ra, dec, z, z_flag_homogenized, instrument_type_homogenized, compared_to`. -""" - )) - - # Prepare base DF with the same rounding used in validation - df_base = df_original.copy() - df_base[source_col] = df_base[source_col].map(_norm_str).astype("string") - df_base["ra4"] = pd.to_numeric(df_base["ra"], errors="coerce").round(ndp_used) - df_base["dec4"] = pd.to_numeric(df_base["dec"], errors="coerce").round(ndp_used) - - cs = res.get("cell_summary") - diag = res.get("diag_sources") - if cs is None or diag is None or getattr(diag, "empty", True): - display(Markdown("_No eligible cells found for samples._")) - else: - # pick sources with at least one cell of 2+ objects - sources_to_show = diag.loc[diag["cells_2plus"] > 0, "source"].tolist() - if max_sources is not None: - sources_to_show = sources_to_show[:max_sources] - - # columns to display (only those present) - col_candidates = [ - "CRD_ID", "ra", "dec", "z", - "z_flag_homogenized", - "instrument_type_homogenized", # preferred - "type_homogenized", # fallback, if it exists - "compared_to", - ] - cols_present = [c for c in col_candidates if c in df_base.columns] - - for src in sources_to_show: - subcells = cs[cs["source"] == src] - if subcells.empty: - continue - - # prioritize largest cells (more pairs => more objects) - subcells_sorted = subcells.sort_values("n_pairs", ascending=False).head(samples_per_source) - - display(Markdown(f"#### Source: `{src}`")) - for _, row in subcells_sorted.iterrows(): - ra4, dec4 = row["ra4"], row["dec4"] - group = df_base[ - (df_base[source_col] == src) & - (df_base["ra4"] == ra4) & - (df_base["dec4"] == dec4) - ].copy() - - # light sorting for readability - if "z_flag_homogenized" in group.columns: - group = group.sort_values(["z_flag_homogenized", "CRD_ID"], ascending=[False, True]) - else: - group = group.sort_values("CRD_ID") - - meta = ( - f"- Cell **ra4={ra4}**, **dec4={dec4}** — " - f"pairs: {int(row['n_pairs'])}, ok_bi: {int(row['n_ok_bi'])}, " - f"any: {int(row['n_ok_any'])}, missing: {int(row['n_missing'])}" - ) - display(Markdown(meta)) - display(group[cols_present].reset_index(drop=True)) - - # --- DIAGNOSTIC BY SOURCE --- - diag = res.get("diag_sources") - display(Markdown( -""" -### Diagnostic by `source` (`res["diag_sources"]`) -- **`source`**: catalog/source. -- **`n_cells`**: total number of cells found in that `source` (includes cells with a single object). -- **`cells_2plus`**: number of cells with **≥ 2 objects** (i.e., generating pairs for validation). -""" - )) - if diag is not None and hasattr(diag, "empty") and not diag.empty: - display(diag.head(top_k)) - else: - display(Markdown("_No rows in `diag_sources`._")) - - # --- CELL SUMMARY --- - cell_summary = res.get("cell_summary") - display(Markdown( -f""" -### Coverage per cell (`res["cell_summary"]`) -- **`source`**: catalog. -- **`ra4`, `dec4`**: RA/DEC rounded to {ndp_used} decimals (10⁻⁴ deg ≈ 0.36″). -- **`n_pairs`**: number of A–B pairs in this cell. -- **`n_ok_bi`**: number of bidirectional pairs. -- **`n_ok_any`**: number of pairs with at least one direction present. -- **`n_missing`**: number of pairs with no direction present. -- **`bi_cov`**: bidirectional coverage = `n_ok_bi / n_pairs`. -- **`any_cov`**: “at least one direction” coverage = `n_ok_any / n_pairs`. -""" - )) - if cell_summary is not None and hasattr(cell_summary, "empty") and not cell_summary.empty: - display(cell_summary.head(top_k)) - else: - display(Markdown("_No rows in `cell_summary`._")) - - # --- PAIRS WITH PROBLEMS --- - viol = res.get("violations") - display(Markdown( -""" -### Non-bidirectional pairs (`res["violations"]`) -- **`A`, `B`**: CRD_IDs of the pair. -- **`A_in_B`**: `True` if A appears in B’s `compared_to`. -- **`B_in_A`**: `True` if B appears in A’s `compared_to`. -- **`status`**: - - `ok_bi` → bidirectional (ideal), - - `ok_one_way` → only one direction present, - - `missing` → no direction present. -""" - )) - if viol is not None and hasattr(viol, "empty") and not viol.empty: - display(viol.head(top_k)) - else: - display(Markdown("_No violations — all pairs are `ok_bi`._")) - -# ======================= -# ======================= -# Cross Sources -# ======================= -# ======================= -# ------------------------------ -# Public API (star-inclusive graph + star consistency) -# ------------------------------ -def validate_tie_results_fast( - df_final: pd.DataFrame, - threshold: float = 0.0005, - max_groups: int | None = None, - include_rows: bool = False, -) -> Dict[str, object]: - """ - Unified-graph validator (stars do not connect). - - * Build a single graph EXCLUDING any edge/node that touches a star. - * Stars for validation are ONLY those with z_flag_homogenized == 6 - (do not infer stars from tie_result == 3). - * Within each component (non-stars only), apply rules among non-stars. - Stars do not connect groups nor influence 1/2. - - Args: - df_final: Input dataframe. - threshold: Delta-z threshold (pairs with Δz <= threshold count as "too close"). - max_groups: Optional cap on scanned components (non-star components). - include_rows: Include violating rows (dataframes) in payload. - - Returns: - Dict with summary and violations. - """ - # --------------------------- - # Input prep - # --------------------------- - df = df_final.copy() - - # Ensure columns exist - req = [ - "CRD_ID", - "compared_to", - "z_flag_homogenized", - "tie_result", - "instrument_type_homogenized", - "z", - ] - for c in req: - if c not in df.columns: - df[c] = np.nan - - # Canonical dtypes for core columns - df["CRD_ID"] = df["CRD_ID"].astype(str) - df["compared_to"] = df["compared_to"].astype("string") - - # Global numeric views - zf_all = pd.to_numeric(df["z_flag_homogenized"], errors="coerce") - tr_all = pd.to_numeric(df["tie_result"], errors="coerce") - - # Stars for VALIDATION = z_flag_homogenized == 6 ONLY - is_star_row = zf_all.eq(6.0) - star_ids = set(df.loc[is_star_row, "CRD_ID"].astype(str)) - - violations: list[dict] = [] - - # --------------------------- - # Global star consistency checks - # --------------------------- - star_must_be_3 = df[is_star_row & ~tr_all.eq(3.0)] - if not star_must_be_3.empty: - violations.append( - { - "rule": "STAR_MUST_BE_3", - "message": "Rows with z_flag_homogenized==6 must have tie_result==3.", - "group_ids": tuple(star_must_be_3["CRD_ID"].astype(str).tolist()), - "rows": star_must_be_3 if include_rows else None, - } - ) - - nonstar_cant_be_3 = df[~is_star_row & tr_all.eq(3.0)] - if not nonstar_cant_be_3.empty: - violations.append( - { - "rule": "NONSTAR_MUST_NOT_BE_3", - "message": "Rows with z_flag_homogenized!=6 must not have tie_result==3.", - "group_ids": tuple(nonstar_cant_be_3["CRD_ID"].astype(str).tolist()), - "rows": nonstar_cant_be_3 if include_rows else None, - } - ) - - # --------------------------- - # Build edges (EXCLUDE stars as A and as B, and drop B not in df) - # --------------------------- - base = df[["CRD_ID", "compared_to"]].copy() - cmp_df = ( - base.assign(_cmp_list=base["compared_to"].map(_parse_cmp_list_fast))[["CRD_ID", "_cmp_list"]] - .explode("_cmp_list", ignore_index=True) - .rename(columns={"CRD_ID": "A", "_cmp_list": "B"}) - ) - - cmp_df["A"] = cmp_df["A"].astype(str) - cmp_df["B"] = cmp_df["B"].astype("string").str.strip() - cmp_df = cmp_df[cmp_df["B"].notna() & (cmp_df["B"] != "")] - - # Keep only edges to B that actually exist in df (prevents phantom NaN nodes) - valid_ids = set(df["CRD_ID"].astype(str)) - if valid_ids: - cmp_df = cmp_df[cmp_df["B"].isin(valid_ids)] - - # EXCLUDE edges that touch a star (either endpoint) - if star_ids: - cmp_df = cmp_df[~cmp_df["A"].isin(star_ids) & ~cmp_df["B"].isin(star_ids)] - - # No edges → no non-star components to validate (still return star stats). - if cmp_df.empty: - summary = { - "n_components": 0, # only among non-stars - "n_pairs": 0, - "n_groups": 0, - "n_violations": len(violations), - "by_rule": (pd.Series([v["rule"] for v in violations]).value_counts().to_dict() if violations else {}), - "n_stars_excluded": int(len(star_ids)), - } - return {"summary": summary, "violations": violations} - - # --------------------------- - # Nodes, undirected unique edges, connected components (non-stars only) - # --------------------------- - nodes = pd.Index(pd.unique(pd.concat([cmp_df["A"], cmp_df["B"]], ignore_index=True))) - id2ix = {cid: i for i, cid in enumerate(nodes)} - ai = cmp_df["A"].map(id2ix).to_numpy() - bi = cmp_df["B"].map(id2ix).to_numpy() - - lo = np.minimum(ai, bi) - hi = np.maximum(ai, bi) - und = np.stack([lo, hi], axis=1) - und = und[und[:, 0] != und[:, 1]] - - if und.size == 0: - # All edges were self-loops (after cleaning) → every node is its own component - n_comp_all = nodes.size - labels = np.arange(nodes.size, dtype=np.int64) - else: - # Unique undirected edges - undv = und.view([("x", und.dtype), ("y", und.dtype)]) - und = np.unique(undv).view(und.dtype).reshape(-1, 2) - - n_nodes = nodes.size - data = np.ones(len(und), dtype=np.int8) - A = coo_matrix((data, (und[:, 0], und[:, 1])), shape=(n_nodes, n_nodes)) - A = A + A.T - n_comp_all, labels = connected_components(A, directed=False, return_labels=True) - - # --------------------------- - # Precompute arrays aligned to `nodes` (non-stars only set) - # --------------------------- - # node_df is a view of df aligned to ordered 'nodes' - node_df = df.set_index(df["CRD_ID"].astype(str), drop=False).reindex(nodes, copy=False) - - zf = pd.to_numeric(node_df["z_flag_homogenized"], errors="coerce").to_numpy() - tr = pd.to_numeric(node_df["tie_result"], errors="coerce").to_numpy() - z = pd.to_numeric(node_df["z"], errors="coerce").to_numpy() - - # Rank for flags (keep the mapping even though stars should not be present here); - # star(6)=-2, NaN=-1, others numeric. - zf_rank = np.where(np.isnan(zf), -1.0, zf) - zf_rank[zf == 6] = -2.0 - - # Instrument type priority (example mapping; adjust if you use a different one) - type_map = {"s": 3, "g": 2, "p": 1} - it = node_df["instrument_type_homogenized"].map(lambda t: type_map.get(_norm_str(t), 0)).astype(np.int16).to_numpy() - - # Non-star mask among nodes (redundant because we filtered edges to avoid stars, - # but kept as a safety net if rows slip in). - nonstar_mask_nodes = (zf != 6) & (tr != 3) - - # --------------------------- - # Iterate components efficiently - # --------------------------- - order = np.argsort(labels, kind="mergesort") - labels_sorted = labels[order] - boundaries = np.flatnonzero(np.r_[True, labels_sorted[1:] != labels_sorted[:-1], True]) - - n_pairs = 0 - n_groups = 0 - comps_scanned = 0 - - for b in range(len(boundaries) - 1): - if (max_groups is not None) and (comps_scanned >= max_groups): - break - - start, end = boundaries[b], boundaries[b + 1] - comp_idx = order[start:end] # node indices (non-star nodes set) - comp_ids_str = tuple(nodes[comp_idx].tolist()) - - rows_payload_all = node_df.iloc[comp_idx].copy() if include_rows else None - - # Non-star guard (should be all non-stars anyway) - ns_mask_local = nonstar_mask_nodes[comp_idx] - ns_idx = comp_idx[ns_mask_local] - - # Components with <2 non-star nodes have no pair/group rules to check - if ns_idx.size < 2: - comps_scanned += 1 - continue - - # Slice arrays for the component (non-stars) - tr_c = np.nan_to_num(tr[ns_idx], nan=-1.0).astype(np.int16) - it_c = it[ns_idx] - zf_c = zf_rank[ns_idx] - z_c = z[ns_idx] - - m = ns_idx.size - if m == 2: - n_pairs += 1 - a, b_ = 0, 1 - tr_a, tr_b = tr_c[a], tr_c[b_] - zf_a, zf_b = zf_c[a], zf_c[b_] - ts_a, ts_b = it_c[a], it_c[b_] - - za, zb = z_c[a], z_c[b_] - dz = np.inf if (np.isnan(za) or np.isnan(zb)) else abs(za - zb) - - vios_local: List[Tuple[str, str]] = [] - if (tr_a in (0, 1)) and (tr_b in (0, 1)) and (tr_a != tr_b): - # winner must dominate in flag or (if equal) in type; - # if still tied in flag+type, Δz must be <= threshold to justify a single winner; - # otherwise this is suspicious. - win_is_a = (tr_a == 1) - win_zf, los_zf = (zf_a, zf_b) if win_is_a else (zf_b, zf_a) - win_ts, los_ts = (ts_a, ts_b) if win_is_a else (ts_b, ts_a) - cond = ( - (win_zf > los_zf) or - (win_zf == los_zf and win_ts > los_ts) or - (win_zf == los_zf and win_ts == los_ts and (dz <= threshold)) - ) - if not cond: - vios_local.append(( - "PAIR_1v0_PRIORITY", - "Winner lacks higher flag/type, and Δz is not <= threshold when flag/type tie persists.", - )) - elif (tr_a == 2) and (tr_b == 2): - # a proper 2–2 requires equal flag/type; and if both z are defined, they should be > threshold apart - cond = (zf_a == zf_b) and (ts_a == ts_b) and (np.isinf(dz) or (dz > threshold)) - if not cond: - vios_local.append(( - "PAIR_2v2_TIE_CONSISTENCY", - "Tie (2,2) requires equal flag/type and Δz>threshold (or undefined z).", - )) - elif (tr_a == 0) and (tr_b == 0): - vios_local.append(( - "PAIR_0v0_SUSPECT", - "Both eliminated (0,0); check upstream logic.", - )) - else: - vios_local.append(( - "PAIR_INVALID_TIE_PATTERN", - f"Unexpected tie_result pattern: ({tr_a},{tr_b}).", - )) - - if vios_local: - for rule, msg in vios_local: - violations.append( - { - "rule": rule, - "message": msg, - "group_ids": comp_ids_str, - "rows": rows_payload_all, - } - ) - - else: - n_groups += 1 - # Survivors among non-stars are 1 or 2 - surv = np.isin(tr_c, (1, 2)) - - # Flag dominance (NaN -> -1 in zf_c; stars never appear here) - max_flag = np.max(zf_c) if zf_c.size else -np.inf - if np.any(surv & (zf_c < max_flag)): - violations.append( - { - "rule": "GROUP_FLAG_DOMINANCE", - "message": "Survivors include members with lower z_flag than group max (rank: star=-2 < NaN=-1 < others).", - "group_ids": comp_ids_str, - "rows": rows_payload_all, - } - ) - - # Type dominance among top-flag - cand_flag = (zf_c == max_flag) - max_type = np.max(it_c[cand_flag]) if np.any(cand_flag) else -1 - if np.any(surv & (it_c < max_type)): - violations.append( - { - "rule": "GROUP_TYPE_DOMINANCE", - "message": "Survivors include members with lower instrument_type than group max among top-flag.", - "group_ids": comp_ids_str, - "rows": rows_payload_all, - } - ) - - # Δz independence among top-flag & top-type survivors - cand = cand_flag & (it_c == max_type) - idx = np.where(cand & surv)[0] - if idx.size >= 2: - zS = z_c[idx] - too_close = False - for i in range(idx.size - 1): - zi = zS[i] - if np.isnan(zi): - continue - d = np.abs(zS[i + 1 :] - zi) - if np.any(~np.isnan(d) & (d <= threshold)): - too_close = True - break - if too_close: - violations.append( - { - "rule": "GROUP_DELTZ_INDEPENDENCE", - "message": "Two survivors are <= threshold apart among max-flag & max-type.", - "group_ids": comp_ids_str, - "rows": rows_payload_all, - } - ) - - # Survivor count (among non-stars) - n_surv = int(np.sum(surv)) - if n_surv == 0: - violations.append( - { - "rule": "GROUP_NO_SURVIVOR_SUSPECT", - "message": "No survivors among non-stars inside a star-excluded component.", - "group_ids": comp_ids_str, - "rows": rows_payload_all, - } - ) - elif n_surv == 1: - only_idx = int(np.where(surv)[0][0]) - if int(tr_c[only_idx]) != 1: - violations.append( - { - "rule": "GROUP_SINGLE_SURVIVOR_MUST_BE_1", - "message": "Exactly one survivor among non-stars but not labeled tie_result == 1.", - "group_ids": comp_ids_str, - "rows": rows_payload_all, - } - ) - else: - if not np.all(tr_c[surv] == 2): - violations.append( - { - "rule": "GROUP_MULTI_SURVIVOR_MUST_BE_2", - "message": "Multiple survivors among non-stars but not all labeled tie_result == 2.", - "group_ids": comp_ids_str, - "rows": rows_payload_all, - } - ) - - comps_scanned += 1 - - summary = { - "n_components": int(n_comp_all), # components among non-stars only - "n_pairs": int(n_pairs), - "n_groups": int(n_groups), - "n_violations": len(violations), - "by_rule": (pd.Series([v["rule"] for v in violations]).value_counts().to_dict() if violations else {}), - "n_stars_excluded": int(len(star_ids)), - } - return {"summary": summary, "violations": violations} - - -def explain_tie_validation_output( - report: dict, - show_per_rule: int = 3, - prefer_cols: list[str] | None = None, -): - """Explain and display results from validate_tie_results_fast(). - - The validator builds a single graph **excluding any edge that touches a star** - (rows where `z_flag_homogenized == 6` or `tie_result == 3`). All rules are - applied only among **non-star** nodes inside each connected component. - Stars must have `tie_result == 3`, never connect groups, and never influence - 1/2 decisions among non-stars. - """ - if prefer_cols is None: - prefer_cols = [ - "CRD_ID", "ra", "dec", "z", - "z_flag_homogenized", - "instrument_type_homogenized", # preferred - "type_homogenized", # optional fallback if present - "tie_result", - "compared_to", - ] - - summary = report.get("summary") or {} - violations = report.get("violations") or [] - - # --------------------------- - # Header + Global Summary - # --------------------------- - display(Markdown("### Tie-results validation")) - display( - Markdown( - f""" -#### Summary -- **Analyzed components (non-stars only)**: `{summary.get("n_components", 0)}` -- **Pairs (size = 2)**: `{summary.get("n_pairs", 0)}` -- **Groups (size ≥ 3)**: `{summary.get("n_groups", 0)}` -- **Total violations**: `{summary.get("n_violations", 0)}` -- **Stars excluded from the graph** (`z_flag_homogenized==6` or `tie_result==3`): `{summary.get("n_stars_excluded", 0)}` -""" - ) - ) - - # --------------------------- - # Count by Rule - # --------------------------- - by_rule = summary.get("by_rule") or {} - display(Markdown("#### Violations by rule")) - if by_rule: - by_rule_df = ( - pd.DataFrame([(k, v) for k, v in by_rule.items()], columns=["rule", "count"]) - .sort_values("count", ascending=False, kind="mergesort") - .reset_index(drop=True) - ) - display(by_rule_df) - else: - display(Markdown("_No violations — everything consistent ✅_")) - - # If there are no violations, stop early - if not violations: - return - - # --------------------------- - # Samples per Rule - # --------------------------- - display(Markdown("### Samples per rule")) - - # Group violations by rule - vio_by_rule: dict[str, list[dict]] = {} - for v in violations: - vio_by_rule.setdefault(v.get("rule", "UNKNOWN_RULE"), []).append(v) - - # Order rules by frequency if available - if by_rule: - ordered_rules = [r for r, _ in sorted(by_rule.items(), key=lambda x: x[1], reverse=True)] - else: - ordered_rules = list(vio_by_rule.keys()) - - for rule in ordered_rules: - V = vio_by_rule.get(rule, []) - display(Markdown(f"### Rule: `{rule}` — {len(V)} occurrence(s)")) - - # Typical message (from the first occurrence) - msg = V[0].get("message", "") if V else "" - if msg: - display(Markdown(f"> _{msg}_")) - - # Show up to `show_per_rule` examples - for i, viol in enumerate(V[:max(0, int(show_per_rule))], start=1): - group_ids = viol.get("group_ids", ()) - rows_obj = viol.get("rows", None) - rows = rows_obj.copy() if isinstance(rows_obj, pd.DataFrame) else pd.DataFrame() - - display(Markdown(f"**Example {i}** — `group_ids`: `{group_ids}`")) - - # Select columns that exist, preserving the preferred order - cols_present = [c for c in prefer_cols if c in rows.columns] - to_show = (rows[cols_present] if cols_present else rows).reset_index(drop=True) - - # Light sorting for readability - sort_cols: list[tuple[str, bool]] = [] - if "z_flag_homogenized" in to_show.columns: - sort_cols.append(("z_flag_homogenized", False)) - if "tie_result" in to_show.columns: - sort_cols.append(("tie_result", False)) - if "CRD_ID" in to_show.columns: - sort_cols.append(("CRD_ID", True)) - - if sort_cols and len(to_show) > 0: - by = [c for c, _ in sort_cols] - asc = [a for _, a in sort_cols] - to_show = to_show.sort_values(by=by, ascending=asc, kind="mergesort") - - display(to_show) - - # --------------------------- - # Final Notes - # --------------------------- - display( - Markdown( - """ -> **Notes** -> -> - **Star handling (graph construction)** -> Any edge that touches a star is removed. Stars are isolated from the -> non-star connected components and must have `tie_result == 3`. -> -> - **`z_flag_homogenized` ranking (non-stars)** -> For non-stars inside a component, flags are ranked as: -> `NaN → -1` (lowest) < `other numeric flags` (their numeric value). -> (Stars are not present in the component graph.) -> -> - **PAIR_1v0_PRIORITY** -> For a 2-node component with tie pattern `(1,0)`, the winner must have a -> strictly higher flag; if equal, a better `instrument_type`; if still tied, -> require `Δz < threshold`. -> -> - **PAIR_2v2_TIE_CONSISTENCY** -> For a 2-node component with tie pattern `(2,2)`, both members must share -> the same flag and `instrument_type`, and `Δz > threshold` (or undefined). -> -> - **GROUP_* rules (size ≥ 3, non-stars only)** -> Check flag/type dominance and Δz independence among survivors within the -> same component. -""" - ) - ) - - -def render_na_compared_to_validation( - df_final: pd.DataFrame, - show_max: int = 10, - cols_to_show: list[str] | None = None, - assert_if_invalid: bool = False, -): - """Render a Markdown report for three rules on `compared_to`-NA rows. - - Rules: - A) If `compared_to` is , then `tie_result` must be 1. - Exception: `tie_result` may be 3 only if `z_flag_homogenized == 6`. - B) (commented out here; kept for reference) - C) Global: `tie_result == 3` => `z_flag_homogenized == 6`, and optionally the converse. - """ - # Pretty display imports (OK outside notebook) - try: - from IPython.display import display, Markdown - except Exception: - def display(x): # type: ignore - pass - def Markdown(x): # type: ignore - return x - - # Required columns - required = {"compared_to", "tie_result", "z_flag_homogenized"} - missing = sorted(required - set(df_final.columns)) - if missing: - raise KeyError(f"Missing required columns for validation: {missing}") - - if cols_to_show is None: - cols_to_show = [ - "CRD_ID", "ra", "dec", "z", "z_flag", "z_err", - "z_flag_homogenized", "instrument_type", "instrument_type_homogenized", - "tie_result", "survey", "source", "compared_to", - ] - - # Normalize compared_to to true NA - df = df_final.copy() - df["compared_to"] = ( - df["compared_to"] - .astype("string").str.strip() - .replace({ - "": pd.NA, "nan": pd.NA, "NaN": pd.NA, "NA": pd.NA, - "": pd.NA, "None": pd.NA, "null": pd.NA - }) - ) - - # Numeric helper - def _num(s): - return pd.to_numeric(s, errors="coerce") - - # ============================================================================= - # Rule A: compared_to is NA -> tie_result must be 1 (except flag==6 -> 3) - # ============================================================================= - na_cmp = df[df["compared_to"].isna()].copy() - - tie_A = _num(na_cmp["tie_result"]).fillna(-1).astype(int) - zf_A = _num(na_cmp["z_flag_homogenized"]).fillna(-1).astype(int) - - valid_A = tie_A.eq(1) | (tie_A.eq(3) & zf_A.eq(6)) - viol_A = na_cmp.loc[~valid_A].copy() - - total_A = int(len(na_cmp)) - ok_A = int(valid_A.sum()) - bad_A = int(len(viol_A)) - - display(Markdown( -f""" -### Validation A: `compared_to` `` ➜ `tie_result` (with exception flag==6) - -- Rows with `compared_to` ``: **{total_A}** -- Valid: **{ok_A}** -- **INVALID:** **{bad_A}** -""" - )) - - tie_disp_A = _num(na_cmp["tie_result"]).astype("Int64").astype("string").fillna("") - zflag_disp_A = _num(na_cmp["z_flag_homogenized"]).astype("Int64").astype("string").fillna("") - ctab_A = pd.crosstab(tie_disp_A, zflag_disp_A, dropna=False) - - display(Markdown("#### Crosstab A: `tie_result` × `z_flag_homogenized` (isolated; showing ``)")) - display(ctab_A) - - if bad_A > 0: - cols_present_A = [c for c in cols_to_show if c in viol_A.columns] - display(Markdown(f"#### ⚠️ Examples (up to {show_max}) — Rule A")) - display(viol_A[cols_present_A].head(show_max).reset_index(drop=True)) - else: - display(Markdown("✅ Rule A OK: all isolated rows follow the rule.")) - - # ============================================================================= - # Rule B (GLOBAL): star consistency - # B1) If tie_result == 3 → z_flag_homogenized must be 6 - # B2) If z_flag_homogenized == 6 → tie_result must be 3 - # ============================================================================= - tie_all = _num(df["tie_result"]) - zf_all = _num(df["z_flag_homogenized"]) - - # B1: tie_result==3 without flag==6 - mask_B1 = tie_all.eq(3) & ~zf_all.eq(6) - viol_B1 = df.loc[mask_B1].copy() - - total_B1 = int(tie_all.eq(3).sum()) - bad_B1 = int(mask_B1.sum()) - ok_B1 = int(total_B1 - bad_B1) - - display(Markdown( -f""" -### Validation B1 (Global): **`tie_result == 3` requires `z_flag_homogenized == 6`** - -- Rows with `tie_result == 3`: **{total_B1}** -- Valid (`flag == 6`): **{ok_B1}** -- **INVALID (`flag != 6` or ``):** **{bad_B1}** -""" - )) - - if bad_B1 > 0: - cols_present_B1 = [c for c in cols_to_show if c in viol_B1.columns] - display(Markdown(f"#### ⚠️ Examples (up to {show_max}) — Rule B1")) - display(viol_B1[cols_present_B1].head(show_max).reset_index(drop=True)) - else: - display(Markdown("✅ Rule B1 OK: every `tie_result == 3` has `flag == 6`.")) - - # B2: flag==6 → tie_result must be 3 - mask_B2 = zf_all.eq(6) & ~tie_all.eq(3) - viol_B2 = df.loc[mask_B2].copy() - - total_B2 = int(zf_all.eq(6).sum()) - bad_B2 = int(mask_B2.sum()) - ok_B2 = int(total_B2 - bad_B2) - - display(Markdown( -f""" -### Validation B2 (Global): **`z_flag_homogenized == 6` requires `tie_result == 3`** - -- Rows with `flag == 6`: **{total_B2}** -- Valid (`tie_result == 3`): **{ok_B2}** -- **INVALID (`tie_result != 3`):** **{bad_B2}** -""" - )) - - if bad_B2 > 0: - cols_present_B2 = [c for c in cols_to_show if c in viol_B2.columns] - display(Markdown(f"#### ⚠️ Examples (up to {show_max}) — Rule B2")) - display(viol_B2[cols_present_B2].head(show_max).reset_index(drop=True)) - else: - display(Markdown("✅ Rule B2 OK: every `flag == 6` has `tie_result == 3`.")) - - # ============================================================================= - # Assertion (optional) — fail if ANY rule is violated - # ============================================================================= - if assert_if_invalid and (bad_A > 0 or bad_B1 > 0 or bad_B2 > 0): - raise AssertionError( - "Validation failed:\n" - f"- Rule A violations: {bad_A}\n" - f"- Rule B1 (tie==3 -> flag==6) violations: {bad_B1}\n" - f"- Rule B2 (flag==6 -> tie==3) violations: {bad_B2}\n" - "See displayed tables for examples." - ) - - return { - # Rule A - "rule_na_cmp_total": total_A, - "rule_na_cmp_valid": ok_A, - "rule_na_cmp_invalid": bad_A, - "rule_na_cmp_crosstab": ctab_A, - "rule_na_cmp_violations": viol_A, - - # Rule B1 (global: tie==3 -> flag==6) - "rule_global_tie3_total": total_B1, - "rule_global_tie3_valid": ok_B1, - "rule_global_tie3_invalid": bad_B1, - "rule_global_tie3_violations": viol_B1, - - # Rule B2 (global: flag==6 -> tie==3) - "rule_global_flag6_total": total_B2, - "rule_global_flag6_valid": ok_B2, - "rule_global_flag6_invalid": bad_B2, - "rule_global_flag6_violations": viol_B2, - } - - - -# ======================= -# ======================= -# Manual Validation -# ======================= -# ======================= -def analyze_groups_by_compared_to_fast( - df_final: pd.DataFrame, - threshold: float = 0.0005, - max_groups: int | None = 10000, - max_examples_per_case: int = 5, - desired_order: list[str] | None = None, - final_columns: list[str] | None = None, - render: bool = True, -) -> dict: - """Fast group analyzer using sparse graph + connected_components. - - Preserves buckets: - - CASE1_* (pairs) and CASE2_* (groups ≥3), with *_small/_large and *_same/_diff - - TIE_FLAG_TYPE_BREAK_PAIR / _GROUP - - SAME_FLAG_DIFF_TYPE - - SAME_SOURCE_PAIR - - Args: - df_final: Input dataframe. - threshold: Δz threshold for small/large classification. - max_groups: Max number of components to scan. - max_examples_per_case: Max examples to render per bucket. - desired_order: Preferred column order in rendering. - final_columns: Final subset of columns for tables. - render: Whether to display Markdown/tables. - - Returns: - Dict with processed_groups, groups_by_case, case_descriptions, summary_counts. - """ - - # --------- display defaults - if desired_order is None: - desired_order = [ - "CRD_ID", "id", "ra", "dec", "z", "z_flag", "z_err", "instrument_type", - "survey", "source", "z_flag_homogenized", "instrument_type_homogenized", - "tie_result", "compared_to", "role", - ] - if final_columns is None: - final_columns = [ - "CRD_ID", "ra", "dec", "z", "z_flag", "z_err", - "z_flag_homogenized", "instrument_type", "instrument_type_homogenized", - "tie_result", "survey", "source", "compared_to", - ] - - # --------- filter rows with non-empty compared_to (string-aware) - df = df_final.copy() - cmp = ( - df[["CRD_ID", "compared_to"]] - .assign(compared_to=lambda x: x["compared_to"].astype("string")) - ) - nonempty_mask = cmp["compared_to"].notna() & (cmp["compared_to"].str.strip() != "") - df_cmp = df.loc[nonempty_mask].copy() - n_rows_with_cmp = int(len(df_cmp)) - if n_rows_with_cmp == 0: - # Nothing to process - out = { - "processed_groups": 0, - "groups_by_case": {k: [] for k in [ - "CASE1_small_same", "CASE1_small_diff", "CASE1_large_same", "CASE1_large_diff", - "CASE2_small_same", "CASE2_small_diff", "CASE2_large_same", "CASE2_large_diff", - "TIE_FLAG_TYPE_BREAK_PAIR", "TIE_FLAG_TYPE_BREAK_GROUP", - "SAME_FLAG_DIFF_TYPE", "SAME_SOURCE_PAIR", - ]}, - "case_descriptions": {}, - "summary_counts": {}, - } - if render: - try: - from IPython.display import display, Markdown - except Exception: - def display(x): pass # type: ignore - def Markdown(x): return x # type: ignore - display(Markdown("### Manual group analysis via `compared_to`")) - display(Markdown("- No rows with non-empty `compared_to`.")) - return out - - # --------- vectorized edges from compared_to - tmp = df_cmp[["CRD_ID", "compared_to"]].copy() - tmp["_nb_list"] = tmp["compared_to"].map(_parse_cmp_list_fast) - edges = ( - tmp[["CRD_ID", "_nb_list"]] - .explode("_nb_list", ignore_index=True) - .rename(columns={"CRD_ID": "A", "_nb_list": "B"}) - ) - edges["A"] = edges["A"].astype(str) - edges["B"] = edges["B"].astype("string").fillna("") - edges = edges[edges["B"].str.strip() != ""] - - if edges.empty: - # No real edges -> no groups - out = { - "processed_groups": 0, - "groups_by_case": {k: [] for k in [ - "CASE1_small_same", "CASE1_small_diff", "CASE1_large_same", "CASE1_large_diff", - "CASE2_small_same", "CASE2_small_diff", "CASE2_large_same", "CASE2_large_diff", - "TIE_FLAG_TYPE_BREAK_PAIR", "TIE_FLAG_TYPE_BREAK_GROUP", - "SAME_FLAG_DIFF_TYPE", "SAME_SOURCE_PAIR", - ]}, - "case_descriptions": {}, - "summary_counts": {}, - } - if render: - try: - from IPython.display import display, Markdown - except Exception: - def display(x): pass # type: ignore - def Markdown(x): return x # type: ignore - display(Markdown("### Manual group analysis via `compared_to`")) - display(Markdown("- No edges after cleaning `compared_to`.")) - return out - - # --------- normalize nodes and deduplicate undirected edges - nodes = pd.Index(pd.unique(pd.concat([edges["A"], edges["B"]], ignore_index=True))) - id2ix = pd.Series(np.arange(nodes.size), index=nodes) # Series for fast map - ai = id2ix.loc[edges["A"]].to_numpy() - bi = id2ix.loc[edges["B"]].to_numpy() - lo = np.minimum(ai, bi) - hi = np.maximum(ai, bi) - und = np.stack([lo, hi], axis=1) - und = und[und[:, 0] != und[:, 1]] - if und.size == 0: - # only self-loops (discarded) - out = { - "processed_groups": 0, - "groups_by_case": {k: [] for k in [ - "CASE1_small_same", "CASE1_small_diff", "CASE1_large_same", "CASE1_large_diff", - "CASE2_small_same", "CASE2_small_diff", "CASE2_large_same", "CASE2_large_diff", - "TIE_FLAG_TYPE_BREAK_PAIR", "TIE_FLAG_TYPE_BREAK_GROUP", - "SAME_FLAG_DIFF_TYPE", "SAME_SOURCE_PAIR", - ]}, - "case_descriptions": {}, - "summary_counts": {}, - } - if render: - try: - from IPython.display import display, Markdown - except Exception: - def display(x): pass # type: ignore - def Markdown(x): return x # type: ignore - display(Markdown("### Manual group analysis via `compared_to`")) - display(Markdown("- Only self-loops; no useful groups.")) - return out - - # deduplicate edges - und_view = und.view([('x', und.dtype), ('y', und.dtype)]) - und = np.unique(und_view).view(und.dtype).reshape(-1, 2) - - # --------- sparse graph and components - n = nodes.size - data = np.ones(len(und), dtype=np.int8) - A = coo_matrix((data, (und[:, 0], und[:, 1])), shape=(n, n)) - A = A + A.T - n_comp, labels = connected_components(A, directed=False, return_labels=True) - - # --------- fast indexing by CRD_ID - base = df_cmp.set_index(df_cmp["CRD_ID"].astype(str), drop=False) - present_mask = nodes.isin(base.index) - comp_ids = [np.where(labels == k)[0] for k in range(n_comp)] - - # --------- quick helpers - def _pairwise_max_and_all_leq(arr: np.ndarray, thr: float) -> tuple[float, bool]: - """Return (max_pairwise_abs_diff, all_pairs_leq_thr) with early-stop.""" - m = arr.size - if m <= 1: - return 0.0, True - max_d = 0.0 - all_leq = True - for i in range(m - 1): - d = np.abs(arr[i+1:] - arr[i]) - md = float(d.max(initial=0.0)) - if md > max_d: - max_d = md - if all_leq and np.any(d > thr): - all_leq = False - return max_d, all_leq - - def _same_survey_nonempty(vals: pd.Series) -> bool: - vs = vals.astype(str).str.strip() - vs = vs[vs != ""] - return vs.nunique(dropna=True) == 1 - - # --------- output buckets - groups_by_case: dict[str, list[pd.DataFrame]] = { - "CASE1_small_same": [], - "CASE1_small_diff": [], - "CASE1_large_same": [], - "CASE1_large_diff": [], - "CASE2_small_same": [], - "CASE2_small_diff": [], - "CASE2_large_same": [], - "CASE2_large_diff": [], - "TIE_FLAG_TYPE_BREAK_PAIR": [], - "TIE_FLAG_TYPE_BREAK_GROUP": [], - "SAME_FLAG_DIFF_TYPE": [], - "SAME_SOURCE_PAIR": [], - } - case_descriptions = { - "CASE1_small_same": f"pair with Δz ≤ {threshold} from the same survey", - "CASE1_small_diff": f"pair with Δz ≤ {threshold} from different surveys", - "CASE1_large_same": f"pair with Δz > {threshold} from the same survey", - "CASE1_large_diff": f"pair with Δz > {threshold} from different surveys", - "CASE2_small_same": f"group (≥3) with all Δz ≤ {threshold} from the same survey", - "CASE2_small_diff": f"group (≥3) with all Δz ≤ {threshold} from different surveys", - "CASE2_large_same": f"group (≥3) with some Δz > {threshold} from the same survey", - "CASE2_large_diff": f"group (≥3) with some Δz > {threshold} from different surveys", - "TIE_FLAG_TYPE_BREAK_PAIR": "pair with equal z_flag_homogenized, different instrument_type_homogenized, different surveys", - "TIE_FLAG_TYPE_BREAK_GROUP": "group (≥3) with equal z_flag_homogenized, different instrument_type_homogenized, different surveys", - "SAME_FLAG_DIFF_TYPE": "group (≥3) with same z_flag_homogenized but at least one differing instrument_type_homogenized", - "SAME_SOURCE_PAIR": "pair with identical source (normalized, non-null)", - } - - processed_groups = 0 - - # --------- linear iteration by component (with early cap) - for k in range(n_comp): - if max_groups is not None and processed_groups >= max_groups: - break - - gidx = comp_ids[k] # global node indices - present = present_mask[gidx] # nodes that exist in df_cmp - # materialize the group with present rows - group_ids = nodes[gidx[present]].tolist() - if len(group_ids) < 2: - continue # ignore singletons - - group = base.loc[group_ids].copy() - - # "role" only to mark the seed row (display) - start_id = group_ids[0] - group["role"] = np.where(group["CRD_ID"].astype(str).eq(start_id), "principal", "compared") - - # quick metrics - z_vals = pd.to_numeric(group["z"], errors="coerce").to_numpy() - max_dz, all_leq = _pairwise_max_and_all_leq(z_vals[np.isfinite(z_vals)], threshold) - same_survey = _same_survey_nonempty(group["survey"]) - - # classification - if len(group) == 2: - key = ( - "CASE1_small_same" if (max_dz <= threshold and same_survey) else - "CASE1_small_diff" if (max_dz <= threshold) else - "CASE1_large_same" if (same_survey) else - "CASE1_large_diff" - ) - else: - key = ( - "CASE2_small_same" if (all_leq and same_survey) else - "CASE2_small_diff" if (all_leq) else - "CASE2_large_same" if (same_survey) else - "CASE2_large_diff" - ) - - # reorder columns for readability - all_columns = list(group.columns) - ordered_columns = (desired_order or []) + [c for c in all_columns if c not in (desired_order or [])] - group = group.reindex(columns=ordered_columns) - groups_by_case[key].append(group) - - # extra buckets - flags = set(group["z_flag_homogenized"].dropna()) - types = set(group["instrument_type_homogenized"].dropna()) - surveys_in_group = set(group["survey"].dropna()) - if len(surveys_in_group) > 1 and len(flags) == 1 and len(types) > 1: - if len(group) == 2: - groups_by_case["TIE_FLAG_TYPE_BREAK_PAIR"].append(group) - else: - groups_by_case["TIE_FLAG_TYPE_BREAK_GROUP"].append(group) - if len(flags) == 1 and len(types) > 1 and len(group) > 2: - groups_by_case["SAME_FLAG_DIFF_TYPE"].append(group) - if len(group) == 2: - src = group["source"].dropna().astype(str).str.strip().str.lower() - if src.size == 2 and src.nunique() == 1: - groups_by_case["SAME_SOURCE_PAIR"].append(group) - - processed_groups += 1 - - # --------- summary - summary_counts = {k: len(v) for k, v in groups_by_case.items()} - - # --------- rendering (with diversity sampling by survey signature) - if render: - try: - from IPython.display import display, Markdown - except Exception: - def display(x): pass # type: ignore - def Markdown(x): return x # type: ignore - - def survey_signature(df: pd.DataFrame) -> tuple[str, ...]: - vals = df["survey"].dropna().astype(str).unique().tolist() - return tuple(sorted(vals)) if len(vals) else ("",) - - display(Markdown(f"### Manual group analysis via `compared_to` (fast)")) - display(Markdown( - f"- Rows with **non-empty** `compared_to`: **{n_rows_with_cmp}** \n" - f"- Unique connected groups processed: **{processed_groups}** \n" - f"- Δz threshold: **{threshold}**" - )) - - if any(summary_counts.values()): - summary_df = pd.DataFrame( - sorted(summary_counts.items(), key=lambda x: (-x[1], x[0])), - columns=["case", "count"] - ) - display(Markdown("#### Groups per bucket")) - display(summary_df.reset_index(drop=True)) - else: - display(Markdown("_No groups found under the current filters._")) - - for case_name, groups in groups_by_case.items(): - if not groups: - continue - # diversity by survey signature - seen_sigs = set() - selection, leftovers = [], [] - for g in groups: - sig = survey_signature(g) - (selection if sig not in seen_sigs else leftovers).append(g) - seen_sigs.add(sig) - if len(selection) >= max_examples_per_case: - break - i = 0 - while len(selection) < max_examples_per_case and i < len(leftovers): - selection.append(leftovers[i]); i += 1 - - desc = case_descriptions.get(case_name, case_name) - display(Markdown(f"#### {case_name} — {desc} \nFound: **{len(groups)}** group(s)")) - for g in selection: - to_show = g.copy() - cols_present = [c for c in (final_columns or []) if c in to_show.columns] - if cols_present: - to_show = to_show.reindex(columns=cols_present) - display(to_show.reset_index(drop=True)) - display(Markdown("---")) - - return { - "processed_groups": processed_groups, - "groups_by_case": groups_by_case, - "case_descriptions": case_descriptions, - "summary_counts": summary_counts, - } - - -def analyze_groups_by_group_id_fast( - df_final: pd.DataFrame, - threshold: float = 0.0005, - max_groups: int | None = 5000, # early cut BEFORE aggregations - max_examples_per_case: int = 4, - render: bool = True, - compute_same_source_pair: bool = False, # turn off to save time - desired_order: list[str] | None = None, - final_columns: list[str] | None = None, -) -> dict: - """ - Ultra-fast manual inspection helper for dedup groups keyed by `group_id`. - - Parameters - ---------- - df_final : pd.DataFrame - Input DataFrame containing, at least, the columns listed in `need`. - threshold : float, default 0.0005 - Delta-z threshold used to classify groups into "small" (<=) vs "large" (>). - max_groups : int | None, default 5000 - Limit on number of groups (with size >= 2) considered for aggregation/rendering. - If None, consider all. - max_examples_per_case : int, default 4 - Max number of example groups rendered per bucket/case. - render : bool, default True - If True, display summary and per-case sample groups. - compute_same_source_pair : bool, default False - If True, additionally compute SAME_SOURCE_PAIR bucket (slower). - desired_order : list[str] | None - Preferred column order for example tables (render only). - final_columns : list[str] | None - Minimal set of columns to display in example tables (render only). - - Returns - ------- - dict - { - "processed_groups": int, - "groups_by_case": dict[str, list[pd.DataFrame]], - "case_descriptions": dict[str, str], - "summary_counts": dict[str, int], - } - """ - # --- minimal columns - need = [ - "group_id", "CRD_ID", "z", "survey", "z_flag_homogenized", - "instrument_type_homogenized", "source", "tie_result", "compared_to", "ra", "dec", - ] - need = [c for c in need if c in df_final.columns] - df = df_final[need].copy() - - # --- fast dtypes - # group_id as nullable integer - df["group_id"] = pd.to_numeric(df["group_id"], errors="coerce").astype("Int64") - valid = df["group_id"].notna() - if not valid.any(): - empty = {k: 0 for k in [ - "CASE1_small_same", "CASE1_small_diff", "CASE1_large_same", "CASE1_large_diff", - "CASE2_small_same", "CASE2_small_diff", "CASE2_large_same", "CASE2_large_diff", - "TIE_FLAG_TYPE_BREAK_PAIR", "TIE_FLAG_TYPE_BREAK_GROUP", - "SAME_FLAG_DIFF_TYPE", "SAME_SOURCE_PAIR", - ]} - return { - "processed_groups": 0, - "groups_by_case": {k: [] for k in empty}, - "case_descriptions": {}, - "summary_counts": empty, - } - - # z as float (only once for valid rows) - zf = pd.to_numeric(df.loc[valid, "z"], errors="coerce").astype("float64") - df.loc[valid, "z"] = zf.values - - # --- SAFE categorical promotion for string-like columns - # NOTE: never cast only a slice to category (it breaks with StringDtype/CSV). - # Cast the entire column after normalizing to StringDtype and (optionally) masking non-valid rows. - for col in ("survey", "z_flag_homogenized", "instrument_type_homogenized"): - if col in df.columns: - # Normalize whole column to StringDtype - df[col] = df[col].astype("string") - # Optional: ensure rows outside 'valid' are NA so categories reflect valid groups - df.loc[~valid, col] = pd.NA - # Cast entire column to category - df[col] = df[col].astype("category") - - # --- pre-cut groups: keep only groups with size >= 2 - gids_all = df.loc[valid, "group_id"] - vc = gids_all.value_counts(sort=False) - gids_ge2 = vc.index[vc.ge(2)] - if len(gids_ge2) == 0: - empty = {k: 0 for k in [ - "CASE1_small_same", "CASE1_small_diff", "CASE1_large_same", "CASE1_large_diff", - "CASE2_small_same", "CASE2_small_diff", "CASE2_large_same", "CASE2_large_diff", - "TIE_FLAG_TYPE_BREAK_PAIR", "TIE_FLAG_TYPE_BREAK_GROUP", - "SAME_FLAG_DIFF_TYPE", "SAME_SOURCE_PAIR", - ]} - return { - "processed_groups": 0, - "groups_by_case": {k: [] for k in empty}, - "case_descriptions": {}, - "summary_counts": empty, - } - - if max_groups is not None and len(gids_ge2) > max_groups: - # Simple truncation: first N (preserve natural order/index) - gids_ge2 = gids_ge2[:max_groups] - - df = df[df["group_id"].isin(gids_ge2)] - - # --- vectorized aggregations - gb = df.groupby("group_id", sort=False) - agg_meta = gb.agg( - size=("CRD_ID", "size"), - z_min=("z", "min"), - z_max=("z", "max"), - survey_nuniq=("survey", "nunique"), - zflag_nuniq=("z_flag_homogenized", "nunique"), - itype_nuniq=("instrument_type_homogenized", "nunique"), - ) - - dz = (agg_meta["z_max"] - agg_meta["z_min"]).astype(float) - all_leq = dz.le(float(threshold)).fillna(False) - same_survey = agg_meta["survey_nuniq"].eq(1) - is_pair = agg_meta["size"].eq(2) - is_group = agg_meta["size"].ge(3) - - # --- case classification - case = pd.Series(index=agg_meta.index, dtype="string") - case.loc[is_pair & all_leq & same_survey] = "CASE1_small_same" - case.loc[is_pair & all_leq & ~same_survey] = "CASE1_small_diff" - case.loc[is_pair & ~all_leq & same_survey] = "CASE1_large_same" - case.loc[is_pair & ~all_leq & ~same_survey] = "CASE1_large_diff" - - case.loc[is_group & all_leq & same_survey] = "CASE2_small_same" - case.loc[is_group & all_leq & ~same_survey] = "CASE2_small_diff" - case.loc[is_group & ~all_leq & same_survey] = "CASE2_large_same" - case.loc[is_group & ~all_leq & ~same_survey] = "CASE2_large_diff" - - tie_mask = ( - agg_meta["zflag_nuniq"].eq(1) - & agg_meta["itype_nuniq"].gt(1) - & agg_meta["survey_nuniq"].gt(1) - ) - tie_pair_ids = set(agg_meta.index[is_pair & tie_mask]) - tie_group_ids = set(agg_meta.index[is_group & tie_mask]) - - samesrc_ids = set() - if compute_same_source_pair and len(tie_pair_ids) != 0: - # Restrict to pairs (cheap) - pair_ids = agg_meta.index[is_pair] - sub = df[df["group_id"].isin(pair_ids)][["group_id", "source"]].copy() - src = sub["source"].astype("string").str.strip().str.lower().fillna("") - nun = sub.assign(_src=src).groupby("group_id")["_src"].nunique(dropna=False) - samesrc_ids = set(nun.index[nun.eq(1)]) - - # --- buckets (sets of group_id) - buckets = { - "CASE1_small_same": set(case.index[case.eq("CASE1_small_same")]), - "CASE1_small_diff": set(case.index[case.eq("CASE1_small_diff")]), - "CASE1_large_same": set(case.index[case.eq("CASE1_large_same")]), - "CASE1_large_diff": set(case.index[case.eq("CASE1_large_diff")]), - "CASE2_small_same": set(case.index[case.eq("CASE2_small_same")]), - "CASE2_small_diff": set(case.index[case.eq("CASE2_small_diff")]), - "CASE2_large_same": set(case.index[case.eq("CASE2_large_same")]), - "CASE2_large_diff": set(case.index[case.eq("CASE2_large_diff")]), - "TIE_FLAG_TYPE_BREAK_PAIR": tie_pair_ids, - "TIE_FLAG_TYPE_BREAK_GROUP": tie_group_ids, - "SAME_FLAG_DIFF_TYPE": set( - agg_meta.index[is_group & agg_meta["zflag_nuniq"].eq(1) & agg_meta["itype_nuniq"].gt(1)] - ), - "SAME_SOURCE_PAIR": samesrc_ids, - } - - summary_counts = {k: len(v) for k, v in buckets.items()} - - # --- case descriptions (also used during rendering) - case_descriptions_local = { - "CASE1_small_same": f"pair with Δz ≤ {threshold} from the same survey", - "CASE1_small_diff": f"pair with Δz ≤ {threshold} from different surveys", - "CASE1_large_same": f"pair with Δz > {threshold} from the same survey", - "CASE1_large_diff": f"pair with Δz > {threshold} from different surveys", - "CASE2_small_same": f"group (≥3) with all Δz ≤ {threshold} from the same survey", - "CASE2_small_diff": f"group (≥3) with all Δz ≤ {threshold} from different surveys", - "CASE2_large_same": f"group (≥3) with some Δz > {threshold} from the same survey", - "CASE2_large_diff": f"group (≥3) with some Δz > {threshold} from different surveys", - "TIE_FLAG_TYPE_BREAK_PAIR": "pair with equal z_flag_homogenized, different instrument_type_homogenized, different surveys", - "TIE_FLAG_TYPE_BREAK_GROUP": "group (≥3) with equal z_flag_homogenized, different instrument_type_homogenized, different surveys", - "SAME_FLAG_DIFF_TYPE": "group (≥3) with same z_flag_homogenized but at least one differing instrument_type_homogenized", - "SAME_SOURCE_PAIR": "pair with identical source (normalized, non-null)", - } - - # --- render examples per bucket - groups_by_case = {k: [] for k in buckets} - - if render and max_examples_per_case > 0: - # Safe defaults for column ordering/selection - if desired_order is None: - desired_order = [ - "CRD_ID", "ra", "dec", "z", "z_flag_homogenized", "instrument_type_homogenized", - "survey", "source", "tie_result", "compared_to", "group_id", - ] - if final_columns is None: - final_columns = [ - "CRD_ID", "ra", "dec", "z", "z_flag_homogenized", "instrument_type_homogenized", - "survey", "source", "tie_result", "compared_to", "group_id", - ] - - sizes = agg_meta["size"] # index = group_id (Int64) - for name, gid_set in buckets.items(): - if not gid_set: - continue - - # Align ids with sizes, then sort by size (desc) - idx = pd.Index(list(gid_set), dtype="Int64") - s = sizes.reindex(idx) - s = s.sort_values(ascending=False, kind="stable") - pick = s.head(max_examples_per_case).index.dropna().tolist() - if not pick: - continue - - sample = df[df["group_id"].isin(pick)].copy() - - # "role": principal = smallest CRD_ID within the group - sample["CRD_ID_str"] = sample["CRD_ID"].astype(str) - first = sample.groupby("group_id")["CRD_ID_str"].transform("min") - sample["role"] = np.where(sample["CRD_ID_str"].eq(first), "principal", "compared") - sample = sample.drop(columns=["CRD_ID_str"]) - - # Reorder columns for readability - ordered = desired_order + [c for c in sample.columns if c not in desired_order] - sample = sample.reindex(columns=ordered) - cols_present = [c for c in final_columns if c in sample.columns] - sample = sample[cols_present + (["role"] if "role" in sample.columns else [])] - - # One DataFrame per group_id - for gid in pick: - gdf = sample[sample["group_id"].eq(gid)].reset_index(drop=True) - groups_by_case[name].append(gdf) - - # --- robust rendering - try: - try: - from IPython.display import display, Markdown # type: ignore - except Exception: - def display(x): print(x) # fallback: plain print - def Markdown(x): return x - - compute_same_source_pair_safe = bool(locals().get("compute_same_source_pair", False)) - - display(Markdown("### Manual group analysis via `group_id` (ultra fast)")) - display(Markdown( - f"- Unique connected groups processed (size ≥ 2): **{int((agg_meta['size']>=2).sum())}** \n" - f"- Δz threshold: **{threshold}** \n" - f"- SAME_SOURCE_PAIR computed: **{compute_same_source_pair_safe}**" - )) - - if any(summary_counts.values()): - summary_df = ( - pd.DataFrame(summary_counts.items(), columns=["case", "count"]) - .sort_values(["count", "case"], ascending=[False, True], ignore_index=True) - ) - display(Markdown("#### Groups per bucket")) - display(summary_df) - else: - display(Markdown("_No groups found under the current filters._")) - - any_examples = False - for case_name, groups in groups_by_case.items(): - if not groups: - continue - any_examples = True - desc = case_descriptions_local.get(case_name, case_name) - display(Markdown(f"#### {case_name} — {desc} \nFound: **{len(groups)}** group(s)")) - for g in groups: - display(g.reset_index(drop=True)) - display(Markdown("---")) - - if not any_examples: - display(Markdown("_No sample groups selected to display (empty buckets or max_examples_per_case=0)._")) - - except Exception as e: - import traceback as _tb - print("ERROR while rendering group samples:", repr(e)) - print(_tb.format_exc()) - - return { - "processed_groups": int((agg_meta["size"] >= 2).sum()), - "groups_by_case": groups_by_case, - "case_descriptions": case_descriptions_local, - "summary_counts": summary_counts, - } - - -def _parse_tokens(s: pd.Series) -> pd.Series: - """Split CSV-like strings into lists of stripped tokens (''/NA -> []).""" - s = s.astype("string") - lst = s.str.split(",") - out = [] - for tokens in lst: - if not tokens: - out.append([]) - continue - toks = [t.strip() for t in tokens if t and t.strip()] - out.append(toks) - return pd.Series(out, index=s.index, dtype="object") - -def _cmp_na_like_mask(df: pd.DataFrame, cmp_col: str) -> pd.Series: - """True when compared_to is NA-like (NA or only whitespace).""" - s = df[cmp_col].astype("string") - return s.isna() | (s.str.strip() == "") - -def _only_star_neighbors_mask( - df_proc: pd.DataFrame, - df_orig: pd.DataFrame, - *, - key: str, - cmp_col: str, - zflag_col: str = "z_flag_homogenized", -) -> pd.Series: - """True when compared_to is non-empty and all neighbors are star IDs - (as defined by df_original[z_flag_homogenized] == 6).""" - if zflag_col not in df_orig.columns: - # If we cannot tell who is a star, be conservative: no rows qualify - return pd.Series(False, index=df_proc.index) - - # Build set of star IDs from original - star_ids = set( - df_orig.loc[ - pd.to_numeric(df_orig[zflag_col], errors="coerce").eq(6.0), key - ].astype("string") - ) - - s = df_proc[cmp_col].astype("string") - non_empty = ~(s.isna() | (s.str.strip() == "")) - tokens = _parse_tokens(s.fillna("")) - - def _all_star(tok_list): - if not tok_list: - return False - return all((t in star_ids) for t in tok_list) - - only_star = tokens.map(_all_star) - return (non_empty & only_star).astype(bool) - -def _check_preserve_block( - df_original: pd.DataFrame, - df_processed: pd.DataFrame, - mask_proc: pd.Series, - *, - key: str, - tr_col: str, - max_examples: int, -) -> Dict: - """Compare tie_result for the subset indicated by mask_proc.""" - proc_sub = df_processed.loc[mask_proc, [key, tr_col]].copy() - proc_sub = proc_sub.rename(columns={tr_col: f"{tr_col}_proc"}) - - orig_min = df_original[[key, tr_col]].rename(columns={tr_col: f"{tr_col}_orig"}) - merged = proc_sub.merge(orig_min, on=key, how="left", validate="m:1") - - missing_in_orig_mask = merged[f"{tr_col}_orig"].isna() & ~merged[f"{tr_col}_proc"].isna() - n_missing_in_original = int(missing_in_orig_mask.sum()) - - tr_proc = pd.to_numeric(merged[f"{tr_col}_proc"], errors="coerce") - tr_orig = pd.to_numeric(merged[f"{tr_col}_orig"], errors="coerce") - equal_mask = (tr_proc == tr_orig) | (tr_proc.isna() & tr_orig.isna()) - mismatch_mask = ~equal_mask & ~missing_in_orig_mask - - mismatches_df = merged.loc[mismatch_mask, [key, f"{tr_col}_proc", f"{tr_col}_orig"]].copy() - if len(mismatches_df) > max_examples: - mismatches_df = mismatches_df.head(max_examples) - - return { - "n_checked": int(len(merged)), - "n_missing_in_original": n_missing_in_original, - "n_mismatches": int(mismatch_mask.sum()), - "mismatches": mismatches_df, - } - -def _check_star_must_be_3( - df_processed: pd.DataFrame, - mask_proc: pd.Series, - *, - key: str, - tr_col: str, - max_examples: int, -) -> dict: - """Check that stars under mask have tie_result == 3. - - Args: - df_processed: Processed DataFrame. - mask_proc: Boolean mask selecting candidate rows. - key: ID column name. - tr_col: Tie result column name. - max_examples: Maximum number of example rows to return. - - Returns: - Dict with counts and example rows where tie_result != 3. - """ - sub = df_processed.loc[mask_proc, [key, tr_col]].copy() - tr_proc = pd.to_numeric(sub[tr_col], errors="coerce") - bad = ~(tr_proc == 3) - - examples = sub.loc[bad, [key, tr_col]].copy() - examples = examples.rename(columns={tr_col: f"{tr_col}_proc"}) - if len(examples) > max_examples: - examples = examples.head(max_examples) - - return { - "n_checked": int(len(sub)), - "n_bad": int(bad.sum()), - "examples": examples, - } - - -def validate_tie_preservation( - df_original: pd.DataFrame, - df_processed: pd.DataFrame, - *, - key: str = "CRD_ID", - tr_col: str = "tie_result", - cmp_col: str = "compared_to", - max_examples: int = 20, -) -> dict: - """Validate tie_result preservation in NA-like and star-neighbor cases. - - Rules: - A) If compared_to is NA-like -> must preserve tie_result. - B) If compared_to has only star neighbors: - - Non-star row -> must preserve tie_result. - - Star row -> must have tie_result == 3. - - Args: - df_original: Original DataFrame. - df_processed: Processed DataFrame. - key: ID column name. - tr_col: Tie result column name. - cmp_col: Compared-to column name. - max_examples: Maximum number of mismatches to show. - - Returns: - Dict with per-case results, totals, and examples. - """ - # A) NA-like case - mask_na = _cmp_na_like_mask(df_processed, cmp_col) - - # B) Only-star neighbors - mask_only_star = _only_star_neighbors_mask( - df_processed, df_original, key=key, cmp_col=cmp_col, zflag_col="z_flag_homogenized" - ) - - # Identify stars in processed - zflag_proc = pd.to_numeric(df_processed.get("z_flag_homogenized"), errors="coerce") - mask_is_star_proc = zflag_proc.eq(6.0) - - # Split into star / non-star cases - mask_only_star_star = (mask_only_star & mask_is_star_proc).fillna(False) - mask_only_star_nonstar = (mask_only_star & ~mask_is_star_proc).fillna(False) - - # Run validations - res_na = _check_preserve_block( - df_original, df_processed, mask_na, key=key, tr_col=tr_col, max_examples=max_examples - ) - res_star_preserve = _check_preserve_block( - df_original, df_processed, mask_only_star_nonstar, key=key, tr_col=tr_col, max_examples=max_examples - ) - res_star_must3 = _check_star_must_be_3( - df_processed, mask_only_star_star, key=key, tr_col=tr_col, max_examples=max_examples - ) - - # Combine examples - mismatches_all = pd.concat( - [res_na["mismatches"], res_star_preserve["mismatches"]], ignore_index=True - ) - if len(mismatches_all) > max_examples: - mismatches_all = mismatches_all.head(max_examples) - - ok = ( - (res_na["n_mismatches"] == 0 and res_na["n_missing_in_original"] == 0) - and (res_star_preserve["n_mismatches"] == 0 and res_star_preserve["n_missing_in_original"] == 0) - and (res_star_must3["n_bad"] == 0) - ) - - return { - "ok": ok, - "na_like": { - "n_checked": res_na["n_checked"], - "n_missing_in_original": res_na["n_missing_in_original"], - "n_mismatches": res_na["n_mismatches"], - }, - "only_star_neighbors_preserve": { - "n_checked": res_star_preserve["n_checked"], - "n_missing_in_original": res_star_preserve["n_missing_in_original"], - "n_mismatches": res_star_preserve["n_mismatches"], - }, - "only_star_neighbors_star3": { - "n_checked": res_star_must3["n_checked"], - "n_bad": res_star_must3["n_bad"], - }, - "n_checked_total": int( - res_na["n_checked"] + res_star_preserve["n_checked"] + res_star_must3["n_checked"] - ), - "n_missing_in_original_total": int( - res_na["n_missing_in_original"] + res_star_preserve["n_missing_in_original"] - ), - "n_mismatches_total": int( - res_na["n_mismatches"] + res_star_preserve["n_mismatches"] - ), - "n_star_must3_bad_total": int(res_star_must3["n_bad"]), - "mismatches": mismatches_all, - "star_must3_examples": res_star_must3["examples"], - } - - -def explain_tie_preservation(result: dict, *, title: str = "Tie-result preservation check") -> str: - """Render human-readable summary of preservation validation results. - - Args: - result: Dict returned by validate_tie_preservation. - title: Report title. - - Returns: - Text summary. - """ - lines = [] - lines.append(f"{title}") - lines.append("-" * len(title)) - - # Case A - lines.append("Case A — compared_to NA-like (preserve tie_result)") - lines.append(f" Checked: {result['na_like']['n_checked']}") - lines.append(f" Missing in original: {result['na_like']['n_missing_in_original']}") - lines.append(f" Mismatches: {result['na_like']['n_mismatches']}") - - # Case B.1 - lines.append("Case B1 — only star neighbors & row is NON-STAR (preserve)") - lines.append(f" Checked: {result['only_star_neighbors_preserve']['n_checked']}") - lines.append(f" Missing in original: {result['only_star_neighbors_preserve']['n_missing_in_original']}") - lines.append(f" Mismatches: {result['only_star_neighbors_preserve']['n_mismatches']}") - - # Case B.2 - lines.append("Case B2 — only star neighbors & row IS STAR (require tie_result==3)") - lines.append(f" Checked: {result['only_star_neighbors_star3']['n_checked']}") - lines.append(f" Not equal to 3: {result['only_star_neighbors_star3']['n_bad']}") - - lines.append("") - lines.append(f"Total checked: {result['n_checked_total']}") - lines.append(f"Total missing in original (preserve-cases): {result['n_missing_in_original_total']}") - lines.append(f"Total mismatches (preserve-cases): {result['n_mismatches_total']}") - lines.append(f"Total star-not-3 (must-be-3 case): {result['n_star_must3_bad_total']}") - lines.append(f"Status: {'OK ✅' if result['ok'] else 'FAIL ❌'}") - - # Examples - mism = result.get("mismatches", None) - if mism is not None and len(mism) > 0: - lines.append("\nExamples (preserve mismatches: proc -> orig):") - for _, row in mism.iterrows(): - lines.append(f" {row.iloc[0]}: {row.iloc[1]} -> {row.iloc[2]}") - - star_bad = result.get("star_must3_examples", None) - if star_bad is not None and len(star_bad) > 0: - lines.append("\nExamples (star must be 3 — proc values):") - for _, row in star_bad.iterrows(): - lines.append(f" {row.iloc[0]}: proc={row.iloc[1]} (expected 3)") - - return "\n".join(lines) diff --git a/packages/crossmatch_auto.py b/packages/crossmatch_auto.py index 1e5092b..a7c8681 100644 --- a/packages/crossmatch_auto.py +++ b/packages/crossmatch_auto.py @@ -25,16 +25,10 @@ # ----------------------- # Third-party # ----------------------- -import numpy as np -import pandas as pd import dask import dask.dataframe as dd -from dask.distributed import get_client - -# ----------------------- -# Project -# ----------------------- -from specz import DTYPE_STR, _build_collection_with_retry +import numpy as np +import pandas as pd from crossmatch_cross import ( _attach_distributed_neighbors, _log_neighbor_saturation_distributed, @@ -48,6 +42,12 @@ project_catalog_for_pair_crossmatch, stage_projected_pairs, ) +from dask.distributed import get_client + +# ----------------------- +# Project +# ----------------------- +from specz import DTYPE_STR, _build_collection_with_retry from utils import get_phase_logger __all__ = ["crossmatch_auto"] diff --git a/packages/crossmatch_cross.py b/packages/crossmatch_cross.py index f2de300..78018db 100644 --- a/packages/crossmatch_cross.py +++ b/packages/crossmatch_cross.py @@ -38,18 +38,14 @@ import time from typing import Dict, Iterable, List, Set -# ----------------------- -# Third-party -# ----------------------- -import numpy as np -import pandas as pd import dask import dask.dataframe as dd # ----------------------- -# Project +# Third-party # ----------------------- -from utils import get_phase_logger +import numpy as np +import pandas as pd from crossmatch_diagnostics import ( log_component_size_diagnostics, log_neighbor_count_diagnostics, @@ -58,16 +54,21 @@ stage_projected_pairs, ) from specz import ( - _build_collection_with_retry, - _normalize_string_series_to_na, - _add_missing_with_dtype, - DTYPE_STR, + DTYPE_BOOL, DTYPE_FLOAT, DTYPE_INT, - DTYPE_BOOL, DTYPE_INT8, + DTYPE_STR, + _add_missing_with_dtype, + _build_collection_with_retry, + _normalize_string_series_to_na, ) +# ----------------------- +# Project +# ----------------------- +from utils import get_phase_logger + __all__ = ["crossmatch_tiebreak", "crossmatch_tiebreak_safe"] LOGGER_NAME = "crc.crossmatch" # child of the pipeline root logger ("crc") diff --git a/packages/crossmatch_diagnostics.py b/packages/crossmatch_diagnostics.py index dc67735..0793e88 100644 --- a/packages/crossmatch_diagnostics.py +++ b/packages/crossmatch_diagnostics.py @@ -2,8 +2,9 @@ from __future__ import annotations -from collections.abc import Mapping, Set as AbstractSet import logging +from collections.abc import Mapping +from collections.abc import Set as AbstractSet import dask.dataframe as dd import numpy as np diff --git a/packages/deduplication.py b/packages/deduplication.py index bcc8d71..77cb952 100644 --- a/packages/deduplication.py +++ b/packages/deduplication.py @@ -10,36 +10,36 @@ # ----------------------- # Standard library # ----------------------- +import hashlib +import logging +import math from typing import ( + Dict, Iterable, + List, Mapping, Sequence, - Dict, - List, ) -import math -import logging -import hashlib + +import dask.dataframe as dd # ----------------------- # Third-party # ----------------------- import numpy as np import pandas as pd -import dask.dataframe as dd REPRESENTATIVE_RADIUS_DIAGNOSTIC_COLUMN = "_diag_representative_max_radius_arcsec" # ----------------------- # Project # ----------------------- -from utils import get_phase_logger, log_phase - from lsdb.dask.merge_catalog_functions import ( + align_and_apply, concat_align_catalogs, get_aligned_pixels_from_alignment, - align_and_apply, ) +from utils import get_phase_logger, log_phase # ----------------------- # Logger (child of 'crc') @@ -76,6 +76,8 @@ def _phase_logger() -> logging.LoggerAdapter: "run_dedup_with_lsdb_map_partitions", "count_global_edge_group_mismatches", "count_global_tie_invariant_violations", + "filter_dask_by_tie_treatment", + "filter_pandas_by_tie_treatment", ] @@ -141,6 +143,10 @@ def _validate_local_tie_invariants( z_flag_col: str = "z_flag_homogenized", ) -> None: """Validate winner/hard-tie semantics for every local component.""" + if z_flag_col not in df.columns: + raise KeyError( + f"Missing required semantic column '{z_flag_col}' for star classification" + ) zf = pd.to_numeric(df[z_flag_col], errors="coerce") tie = pd.to_numeric(df[tie_col], errors="coerce") stars = zf.eq(6.0) @@ -273,7 +279,8 @@ def filter_pandas_by_tie_treatment( """Apply final winner/hard-tie policy to an in-memory result frame. Returns the filtered frame, the effective option, and the number of - hard-tie groups resolved by ``draw_one``. + hard-tie groups resolved by ``draw_one``. Selection is deterministic by + the smallest ``CRD_ID`` when that column is available. """ if tie_col not in df.columns: raise RuntimeError(f"Expected '{tie_col}' column for remove-duplicates mode") @@ -302,13 +309,23 @@ def filter_pandas_by_tie_treatment( candidates.to_numpy(dtype=bool, na_value=False) ) if candidate_positions.size: - rng = np.random.default_rng(random_state) winners = [] candidate_groups = gid.iloc[candidate_positions] for _, positions in pd.Series( candidate_positions, index=candidate_groups.to_numpy() ).groupby(level=0, sort=False): - winners.append(int(rng.choice(positions.to_numpy()))) + group_positions = positions.to_numpy(dtype=int) + if "CRD_ID" in df.columns: + winners.append( + min( + group_positions, + key=lambda pos: str(df.iloc[int(pos)]["CRD_ID"]), + ) + ) + else: + # Preserve the former API for frames without CRD_ID. + rng = np.random.default_rng(random_state) + winners.append(int(rng.choice(group_positions))) resolved_groups = len(winners) tie_num = tie_num.copy() tie_num.iloc[candidate_positions] = 0 @@ -319,6 +336,56 @@ def filter_pandas_by_tie_treatment( return out.loc[keep_mask].copy(), effective, resolved_groups +def filter_dask_by_tie_treatment( + df: dd.DataFrame, + option: str, + *, + tie_col: str = "tie_result", + group_col: str = "group_id", + crd_col: str = "CRD_ID", +) -> tuple[dd.DataFrame, str]: + """Apply the final tie policy lazily to a distributed dataframe. + + ``draw_one`` deterministically selects the lexicographically smallest + ``CRD_ID`` from each hard-tie group. Only hard-tie candidates participate + in the groupby and merge; the full catalog is never collected by the + driver. + """ + if tie_col not in df.columns: + raise RuntimeError(f"Expected '{tie_col}' column for remove-duplicates mode") + + effective = str(option or "remove_all").strip().lower() + if effective not in {"remove_all", "keep_all", "draw_one"}: + effective = "remove_all" + + tie_num = dd.to_numeric(df[tie_col], errors="coerce").fillna(0).astype("int8") + if effective == "keep_all": + return df.loc[tie_num.isin([1, 2])], effective + if effective == "remove_all": + return df.loc[tie_num.eq(1)], effective + + if group_col not in df.columns or crd_col not in df.columns: + return df.loc[tie_num.eq(1)], "remove_all" + + ordinary_winners = df.loc[tie_num.eq(1)] + hard_candidates = df.loc[tie_num.eq(2) & ~df[group_col].isna()] + winner_ids = ( + hard_candidates[[group_col, crd_col]] + .groupby(group_col)[crd_col] + .min() + .rename("__draw_one_crd") + .to_frame() + .reset_index() + ) + selected_hard = hard_candidates.merge( + winner_ids, + left_on=[group_col, crd_col], + right_on=[group_col, "__draw_one_crd"], + how="inner", + ).drop(columns=["__draw_one_crd"]) + return dd.concat([ordinary_winners, selected_hard]), effective + + # ----------------------- # Graph building # ----------------------- @@ -965,7 +1032,7 @@ def deduplicate_pandas( Returns: pd.DataFrame: Deduplicated dataframe with tie labels. """ - required = {crd_col, compared_col, z_col} + required = {crd_col, compared_col, z_col, "z_flag_homogenized"} missing = sorted(required - set(df.columns)) if missing: raise KeyError(f"Missing required columns: {missing}") @@ -1041,6 +1108,13 @@ def deduplicate_pandas( # Try to bridge NA rows via neighbor groups from the fast path label map if na_mask.any() and labels_edge.size: pos_na = np.flatnonzero(na_mask) + # Stars never participate in non-star components, even when they point + # to an already-labelled neighbor. Leave them for the singleton path. + if zf_series is not None: + bridge_is_star = np.asarray( + zf_series.iloc[pos_na].eq(6).fillna(False), dtype=bool + ) + pos_na = pos_na[~bridge_is_star] cmp_str_all = out[compared_col].astype("string").str.strip() cmp_lists = cmp_str_all.iloc[pos_na].str.split(",") sub = pd.DataFrame({"pos": pos_na, "nbr": cmp_lists}).explode( @@ -1472,10 +1546,18 @@ def _dedup_local_with_margin( mg = _to_pandas(part_margin) # Project to required columns. - needed = {crd_col, compared_col, z_col, tie_col, "ra", "dec"} | set( + needed = { + crd_col, + compared_col, + z_col, + tie_col, + "ra", + "dec", + "z_flag_homogenized", + } | set( tiebreaking_priority or [] ) - if instrument_type_priority is not None: + if "instrument_type_homogenized" in set(tiebreaking_priority or []): needed.add("instrument_type_homogenized") pm = _shrink_to_needed(pm, needed, crd_col, compared_col, z_col) mg = _shrink_to_needed(mg, needed, crd_col, compared_col, z_col) @@ -1727,10 +1809,18 @@ def _dedup_local_no_margin( pm = _to_pandas(part_main) # Project to required columns. - needed = {crd_col, compared_col, z_col, tie_col, "ra", "dec"} | set( + needed = { + crd_col, + compared_col, + z_col, + tie_col, + "ra", + "dec", + "z_flag_homogenized", + } | set( tiebreaking_priority or [] ) - if instrument_type_priority is not None: + if "instrument_type_homogenized" in set(tiebreaking_priority or []): needed.add("instrument_type_homogenized") pm = _shrink_to_needed(pm, needed, crd_col, compared_col, z_col) @@ -1880,7 +1970,12 @@ def run_dedup_with_lsdb_map_partitions( ) # Strict schema checks: required base + all tiebreaking priority columns - required_base = {crd_col, compared_col, z_col} + required_base = { + crd_col, + compared_col, + z_col, + "z_flag_homogenized", + } _assert_required(main_ddf, required_base, "main") _assert_priorities(main_ddf, list(tiebreaking_priority), "main") diff --git a/packages/executor.py b/packages/executor.py index 8e8b2ca..4c3b2ac 100644 --- a/packages/executor.py +++ b/packages/executor.py @@ -10,6 +10,7 @@ # Standard library # ----------------------- import logging +import math from typing import Any, Dict # ----------------------- @@ -58,6 +59,8 @@ def get_executor(executor_config: Dict[str, Any], logs_dir: str | None = None): - scale: * minimum_jobs (int) * maximum_jobs (int) + * adaptive_interval_seconds (float) + * adaptive_scale_down_delay_seconds (float) logs_dir: Directory to store SLURM job logs (if provided). Returns: @@ -104,6 +107,28 @@ def get_executor(executor_config: Dict[str, Any], logs_dir: str | None = None): min_jobs = int(scale_cfg.get("minimum_jobs", 0)) max_jobs = int(scale_cfg.get("maximum_jobs", 0) or 0) + adaptive_interval_seconds = float( + scale_cfg.get("adaptive_interval_seconds", 10.0) + ) + adaptive_scale_down_delay_seconds = float( + scale_cfg.get("adaptive_scale_down_delay_seconds", 180.0) + ) + # Keep direct executor callers using the former low-level setting + # working while exposing the clearer duration-based configuration. + if "adaptive_scale_down_delay_seconds" not in scale_cfg and ( + "adaptive_wait_count" in scale_cfg + ): + adaptive_wait_count = int(scale_cfg["adaptive_wait_count"]) + adaptive_scale_down_delay_seconds = ( + adaptive_wait_count * adaptive_interval_seconds + ) + else: + adaptive_wait_count = max( + 1, + math.ceil( + adaptive_scale_down_delay_seconds / adaptive_interval_seconds + ), + ) # n_workers in SLURMCluster = total worker processes n_workers_init = min_jobs * processes @@ -127,11 +152,22 @@ def get_executor(executor_config: Dict[str, Any], logs_dir: str | None = None): # Keep adaptive scaling enabled even when min == max. In that case it # acts as a fixed-size allocation that can replace jobs lost mid-run. - cluster.adapt(minimum_jobs=min_jobs, maximum_jobs=max_jobs) + cluster.adapt( + minimum_jobs=min_jobs, + maximum_jobs=max_jobs, + interval=f"{adaptive_interval_seconds:g}s", + wait_count=adaptive_wait_count, + ) logger.info( - "Adaptive scaling enabled: minimum_jobs=%d maximum_jobs=%d", + "Adaptive scaling enabled: minimum_jobs=%d maximum_jobs=%d " + "interval=%.1fs scale_down_delay=%.1fs wait_count=%d " + "effective_scale_down_delay=%.1fs", min_jobs, max_jobs, + adaptive_interval_seconds, + adaptive_scale_down_delay_seconds, + adaptive_wait_count, + adaptive_wait_count * adaptive_interval_seconds, ) return cluster diff --git a/packages/product_handle.py b/packages/product_handle.py index 99a42e2..bc4916c 100644 --- a/packages/product_handle.py +++ b/packages/product_handle.py @@ -15,8 +15,8 @@ import io import json import os -from importlib.metadata import PackageNotFoundError, version import shutil +from importlib.metadata import PackageNotFoundError, version from pathlib import Path # ----------------------- @@ -957,6 +957,14 @@ def save_dataframe( ext = format_.lower() if ext == "parquet": + if _is_dask_dataframe(df): + _write_single_parquet_from_dask( + df, + Path(f"{output_path}.parquet"), + temp_dir=temp_dir, + logger=logger, + ) + return try: df2 = df.reset_index(drop=True) except Exception: @@ -985,6 +993,15 @@ def save_dataframe( return # CSV/FITS start from numpy_nullable to stabilize null semantics + if ext == "csv" and _is_dask_dataframe(df): + _write_single_csv_from_dask( + df, + Path(f"{output_path}.csv"), + temp_dir=temp_dir, + logger=logger, + ) + return + df_np = df.convert_dtypes(dtype_backend="numpy_nullable", convert_boolean=False) if ext == "csv": @@ -1249,11 +1266,112 @@ def _ensure_staged_parquet_non_empty(parquet_path: Path) -> None: return raise RuntimeError( - "Final catalog is empty after HATS parquet staging. " + "Final catalog is empty after Parquet staging. " "Nothing can be consolidated." ) +def _stage_distributed_single_file_input( + data, + output_path: Path, + temp_dir, + logger, +) -> Path: + """Materialize a Dask dataframe once as distributed Parquet parts.""" + staging_root = Path(temp_dir) if temp_dir else output_path.parent / "temp" + parquet_path = staging_root / f".{output_path.name}_parts" + if parquet_path.exists(): + shutil.rmtree(parquet_path, ignore_errors=True) + parquet_path.mkdir(parents=True, exist_ok=True) + _log_info(logger, "Staging distributed single-file output at %s", parquet_path) + data.to_parquet( + str(parquet_path), + engine="pyarrow", + write_index=False, + overwrite=True, + ) + _ensure_staged_parquet_non_empty(parquet_path) + return parquet_path + + +def _write_single_parquet_from_dask( + data, + output_path: Path, + *, + temp_dir, + logger, +) -> None: + """Write one Parquet file after distributed staging, one batch at a time.""" + parquet_path = _stage_distributed_single_file_input( + data, output_path, temp_dir, logger + ) + parts = sorted(parquet_path.rglob("*.parquet")) + output_path.parent.mkdir(parents=True, exist_ok=True) + temporary_output = output_path.with_name(f".{output_path.name}.tmp") + temporary_output.unlink(missing_ok=True) + + writer = None + try: + for part_path in parts: + source = pq.ParquetFile(part_path) + if writer is None: + writer = pq.ParquetWriter(temporary_output, source.schema_arrow) + for batch in source.iter_batches(batch_size=65_536): + writer.write_table(pa.Table.from_batches([batch])) + if writer is None: + raise RuntimeError("Distributed Parquet staging produced no readable parts") + writer.close() + writer = None + temporary_output.replace(output_path) + except Exception: + if writer is not None: + writer.close() + temporary_output.unlink(missing_ok=True) + raise + else: + shutil.rmtree(parquet_path, ignore_errors=True) + _log_info(logger, "Finished single Parquet output: %s", output_path) + + +def _write_single_csv_from_dask( + data, + output_path: Path, + *, + temp_dir, + logger, +) -> None: + """Write one CSV after distributed staging, one Arrow batch at a time.""" + parquet_path = _stage_distributed_single_file_input( + data, output_path, temp_dir, logger + ) + parts = sorted(parquet_path.rglob("*.parquet")) + output_path.parent.mkdir(parents=True, exist_ok=True) + temporary_output = output_path.with_name(f".{output_path.name}.tmp") + temporary_output.unlink(missing_ok=True) + + wrote_header = False + try: + with temporary_output.open("w", encoding="utf-8", newline="") as handle: + for part_path in parts: + source = pq.ParquetFile(part_path) + for batch in source.iter_batches(batch_size=65_536): + batch.to_pandas().to_csv( + handle, + index=False, + header=not wrote_header, + ) + wrote_header = True + if not wrote_header: + raise RuntimeError("Distributed Parquet staging produced no CSV rows") + temporary_output.replace(output_path) + except Exception: + temporary_output.unlink(missing_ok=True) + raise + else: + shutil.rmtree(parquet_path, ignore_errors=True) + _log_info(logger, "Finished single CSV output: %s", output_path) + + def build_collection_with_retry( parquet_path, logs_dir=None, diff --git a/packages/specz.py b/packages/specz.py index 3fd8b28..c38acce 100644 --- a/packages/specz.py +++ b/packages/specz.py @@ -27,7 +27,8 @@ import dask.dataframe as dd import numpy as np import pandas as pd -from dask.distributed import get_client as _get_client, wait +from dask.distributed import get_client as _get_client +from dask.distributed import wait dask.config.set({"dataframe.shuffle.method": "tasks"}) os.environ.setdefault("DASK_DISTRIBUTED__SHUFFLE__METHOD", "tasks") @@ -38,15 +39,17 @@ import hats # noqa: F401 from product_handle import ( ProductHandle, +) +from product_handle import ( build_collection_with_retry as _build_collection_with_retry, ) -from utils import ensure_crc_logger from specz_homogenization import ( JADES_LETTER_TO_SCORE, VIMOS_FLAG_TO_SCORE, - _honor_user_homogenized_mapping, _homogenize, + _honor_user_homogenized_mapping, ) +from utils import ensure_crc_logger if TYPE_CHECKING: from dask.distributed import Client # noqa: F401 @@ -2330,6 +2333,127 @@ def _maybe_collection( # ----------------------- # Main orchestrator # ----------------------- +def _requires_z_flag_homogenization(combine_mode: str, cut_value: object) -> bool: + """Whether preparation needs the semantic flag outside ranking priorities.""" + if combine_mode in { + "concatenate_and_mark_duplicates", + "concatenate_and_remove_duplicates", + }: + return True + try: + numeric_cut = float(cut_value) + except (TypeError, ValueError): + return False + return numeric_cut in {1.0, 2.0, 3.0, 4.0, 5.0, 6.0} + + +def validate_combine_configuration( + combine_mode: object, + tiebreaking_priority: object, + cut_value: object, + logger: logging.LoggerAdapter | logging.Logger | None = None, +) -> tuple[str, list[str]]: + """Validate combine semantics before any catalog preparation starts.""" + normalized_mode = str(combine_mode or "").strip().lower() + valid_modes = { + "concatenate", + "concatenate_and_mark_duplicates", + "concatenate_and_remove_duplicates", + } + if normalized_mode not in valid_modes: + raise ValueError( + f"Invalid combine_type={combine_mode!r}; expected one of " + f"{sorted(valid_modes)}" + ) + + if tiebreaking_priority is None: + priorities: list[str] = [] + elif isinstance(tiebreaking_priority, (list, tuple)): + priorities = [str(value).strip() for value in tiebreaking_priority] + else: + raise TypeError("tiebreaking_priority must be a list of column names") + if any(not value for value in priorities): + raise ValueError("tiebreaking_priority cannot contain empty column names") + if len(set(priorities)) != len(priorities): + raise ValueError("tiebreaking_priority cannot contain duplicate columns") + if normalized_mode != "concatenate" and not priorities: + raise ValueError( + f"tiebreaking_priority must be non-empty for {normalized_mode}" + ) + + try: + numeric_cut = float(cut_value) if cut_value is not None else None + except (TypeError, ValueError): + numeric_cut = None + if ( + normalized_mode == "concatenate_and_remove_duplicates" + and numeric_cut == 6.0 + and logger is not None + ): + logger.warning( + "z_flag_homogenized_value_to_cut=6 retains only stars, while " + "concatenate_and_remove_duplicates excludes tie_result=3; the final " + "catalog will normally be empty." + ) + return normalized_mode, priorities + + +def _normalize_custom_tiebreaking_priorities( + df: dd.DataFrame, + priorities: list[str], + product_name: str, + logger: logging.LoggerAdapter | logging.Logger, +) -> dd.DataFrame: + """Require and normalize generic ranking columns for one input catalog.""" + custom_priorities = [ + column + for column in priorities + if column + not in {"z_flag_homogenized", "instrument_type_homogenized"} + ] + for column in custom_priorities: + if column not in df.columns: + raise ValueError( + f"[{product_name}] Custom tiebreaking priority '{column}' is " + "missing; custom priorities must exist in every input catalog" + ) + + original = df[column] + numeric = dd.to_numeric(original, errors="coerce") + total_rows, original_non_null, valid_numeric, coerced_invalid = dask.compute( + df.map_partitions(len).sum(), + (~original.isna()).sum(), + (~numeric.isna()).sum(), + ((~original.isna()) & numeric.isna()).sum(), + ) + total_rows = int(total_rows) + original_non_null = int(original_non_null) + valid_numeric = int(valid_numeric) + coerced_invalid = int(coerced_invalid) + if valid_numeric == 0: + raise ValueError( + f"[{product_name}] Custom tiebreaking priority '{column}' has " + "no valid numeric values" + ) + invalid_fraction = ( + coerced_invalid / original_non_null if original_non_null else 0.0 + ) + logger.info( + "[%s] Custom priority '%s': rows=%d valid_numeric=%d " + "missing=%d coerced_invalid=%d invalid_fraction=%.6f; " + "normalized_dtype=float64", + product_name, + column, + total_rows, + valid_numeric, + total_rows - original_non_null, + coerced_invalid, + invalid_fraction, + ) + df = df.assign(**{column: numeric.astype("float64")}) + return df + + def prepare_catalog( entry: dict, translation_config: dict, @@ -2370,6 +2494,10 @@ def prepare_catalog( extra_columns = _normalize_extra_columns_config(param_config.get("extra_columns")) + z_flag_homogenized_value_to_cut = param_config.get( + "z_flag_homogenized_value_to_cut", None + ) + # 1) Load product ph = ProductHandle(entry["path"]) df = ph.to_ddf() @@ -2398,12 +2526,25 @@ def prepare_catalog( tiebreaking_priority, instrument_type_priority, # noqa: F841 translation_rules_uc, - ) = _homogenize(df, translation_config, product_name, lg, type_cast_ok=type_cast_ok) + ) = _homogenize( + df, + translation_config, + product_name, + lg, + type_cast_ok=type_cast_ok, + require_z_flag_homogenized=_requires_z_flag_homogenization( + combine_mode, + z_flag_homogenized_value_to_cut, + ), + ) + df = _normalize_custom_tiebreaking_priorities( + df, + list(tiebreaking_priority), + product_name, + lg, + ) # 7) Apply cut based on z_flag_homogenized if requested - z_flag_homogenized_value_to_cut = param_config.get( - "z_flag_homogenized_value_to_cut", None - ) if ( z_flag_homogenized_value_to_cut is not None and "z_flag_homogenized" in df.columns @@ -2416,10 +2557,14 @@ def prepare_catalog( z_flag_homogenized_value_to_cut, ) else: - if cut_val not in {1.0, 2.0, 3.0, 4.0, 5.0, 6.0}: + if cut_val == 0.0: + # Zero is the explicit no-cut sentinel. It is summarized once + # by the driver instead of repeated for every input catalog. + pass + elif cut_val not in {1.0, 2.0, 3.0, 4.0, 5.0, 6.0}: lg.warning( - "Invalid z_flag_homogenized_value_to_cut=%s; valid cut values " - "are 1, 2, 3, 4, 5, 6. Skipping cut.", + "Invalid z_flag_homogenized_value_to_cut=%s; use 0 to disable " + "the cut or one of 1, 2, 3, 4, 5, 6. Skipping cut.", z_flag_homogenized_value_to_cut, ) else: diff --git a/packages/specz_homogenization.py b/packages/specz_homogenization.py index deefd9d..1c2711a 100644 --- a/packages/specz_homogenization.py +++ b/packages/specz_homogenization.py @@ -194,6 +194,8 @@ def _homogenize( product_name: str, logger: logging.Logger, type_cast_ok: bool, + *, + require_z_flag_homogenized: bool = False, ) -> tuple[dd.DataFrame, bool, list, dict, dict]: """Compute homogenized columns for tie-breaking. @@ -203,6 +205,8 @@ def _homogenize( product_name: Catalog identifier. logger: Logger. type_cast_ok: Whether `type` was normalized. + require_z_flag_homogenized: Create and validate the flag for semantic + star classification even when it is not a ranking priority. Returns: Tuple: (df, used_type_fastpath, tiebreaking_priority, instrument_type_priority, translation_rules_uc) @@ -211,6 +215,8 @@ def _homogenize( instrument_type_priority = translation_config.get("instrument_type_priority", {}) translation_rules_uc = {k.upper(): v for k, v in translation_config.get("translation_rules", {}).items()} validated_non_null_counts: dict[str, int] = {} + z_flag_is_priority = "z_flag_homogenized" in tiebreaking_priority + needs_z_flag = z_flag_is_priority or require_z_flag_homogenized # ----------------------- # Vectorized translator @@ -436,7 +442,7 @@ def quality_like_to_flag(x): return 4.0 return np.nan - if "z_flag_homogenized" in tiebreaking_priority: + if needs_z_flag: if "z_flag_homogenized" not in df.columns: if can_use_zflag_as_quality(): logger.info(f"{product_name} Using 'z_flag' fast path for z_flag_homogenized.") @@ -461,7 +467,7 @@ def quality_like_to_flag(x): # User-provided 'z_flag_homogenized' is present. Validate allowed domain {0,1,2,3,4} (NaN allowed). logger.info(f"{product_name} 'z_flag_homogenized' already exists; validating user-provided values.") allowed = {0.0, 1.0, 2.0, 3.0, 4.0, 6.0} - + vals = dd.to_numeric(df["z_flag_homogenized"], errors="coerce") # NaN is allowed; only non-NaN values outside the allowed set are invalid invalid_mask = (~dd.isna(vals)) & ~vals.isin(list(allowed)) @@ -469,14 +475,14 @@ def quality_like_to_flag(x): invalid_mask.sum(), vals.count() ) validated_non_null_counts["z_flag_homogenized"] = int(non_null_count) - + if invalid_count > 0: examples = df["z_flag_homogenized"].loc[invalid_mask].head(5, compute=True).tolist() raise ValueError( f"[{product_name}] Invalid values in user-provided 'z_flag_homogenized'. " f"Allowed set is {sorted(allowed)} (NaN allowed). Examples of invalid values: {examples}" ) - + # Cast to Arrow-backed float dtype for consistency df["z_flag_homogenized"] = vals.map_partitions( lambda s: s.astype(DTYPE_FLOAT), @@ -531,12 +537,12 @@ def can_use_type_for_instrument() -> bool: # User-provided 'instrument_type_homogenized' is present. Validate allowed domain {"s","p","g"}. logger.info(f"{product_name} 'instrument_type_homogenized' already exists; validating user-provided values.") allowed = {"s", "p", "g"} - + normed = df["instrument_type_homogenized"].map_partitions( _normalize_string_series_to_na, meta=pd.Series(pd.array([], dtype=DTYPE_STR)), ).str.lower() - + invalid_mask = (~dd.isna(normed)) & ~normed.isin(list(allowed)) invalid_count, non_null_count = dask.compute( invalid_mask.sum(), normed.count() @@ -544,32 +550,34 @@ def can_use_type_for_instrument() -> bool: validated_non_null_counts["instrument_type_homogenized"] = int( non_null_count ) - + if invalid_count > 0: examples = df["instrument_type_homogenized"].loc[invalid_mask].head(5, compute=True).tolist() raise ValueError( f"[{product_name}] Invalid values in user-provided 'instrument_type_homogenized'. " f"Allowed set is {sorted(allowed)}. Examples of invalid values: {examples}" ) - + # Keep normalized lower-case values for consistency df["instrument_type_homogenized"] = normed # --- post-homogenization sanity checks (required columns must not be all-NaN) --- - if "z_flag_homogenized" in tiebreaking_priority: + if needs_z_flag: if "z_flag_homogenized" not in df.columns: raise ValueError( - f"[{product_name}] 'z_flag_homogenized' is required by tiebreaking_priority but is missing after homogenization." - ) - non_null = validated_non_null_counts.get("z_flag_homogenized") - if non_null is None: - non_null = dask.compute(df["z_flag_homogenized"].count())[0] - if int(non_null) == 0: - raise ValueError( - f"[{product_name}] All values in 'z_flag_homogenized' are NaN. " - "This column is required (in tiebreaking_priority) and must contain at least one non-NaN value. " - "Verify YAML translations / fast-path logic and input columns." + f"[{product_name}] 'z_flag_homogenized' is required for " + "star classification but is missing after homogenization." ) + if z_flag_is_priority: + non_null = validated_non_null_counts.get("z_flag_homogenized") + if non_null is None: + non_null = dask.compute(df["z_flag_homogenized"].count())[0] + if int(non_null) == 0: + raise ValueError( + f"[{product_name}] All values in 'z_flag_homogenized' are NaN. " + "This column is required (in tiebreaking_priority) and must contain at least one non-NaN value. " + "Verify YAML translations / fast-path logic and input columns." + ) if "instrument_type_homogenized" in tiebreaking_priority: if "instrument_type_homogenized" not in df.columns: diff --git a/packages/utils.py b/packages/utils.py index 4ad6d81..039de29 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -52,8 +52,7 @@ # Third-party # ----------------------- import pandas as pd # kept for other utilities in this module -import yaml # kept for other utilities in this module - +import yaml # kept for other utilities in this module # ----------------------- # Globals @@ -299,7 +298,7 @@ def _in_dask_worker() -> bool: if isinstance(lvl_candidate, int): level = lvl_candidate # --- END: env override for log level --- - + logger.setLevel(level) logger.propagate = False @@ -373,7 +372,7 @@ def _in_dask_worker() -> bool: logging.getLogger().setLevel(logging.WARNING) logging.getLogger("dask").setLevel(logging.WARNING) logging.getLogger("distributed").setLevel(logging.WARNING) - logging.getLogger("lsdb").setLevel(logging.INFO) + logging.getLogger("lsdb").setLevel(logging.INFO) logging.getLogger("urllib3").setLevel(logging.WARNING) # Mark configured for this PID. diff --git a/packages/worker_health.py b/packages/worker_health.py index a8b3bf1..74edab6 100644 --- a/packages/worker_health.py +++ b/packages/worker_health.py @@ -118,4 +118,3 @@ def stop(self) -> None: self._stop_event.set() if self._thread is not None and self._thread is not threading.current_thread(): self._thread.join(timeout=max(1.0, self.check_interval_seconds * 2)) - diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index f5a619b..0000000 --- a/pytest.ini +++ /dev/null @@ -1,4 +0,0 @@ -[pytest] -addopts = -m "not slow" -markers = - slow: integration tests that run the full pipeline and are excluded by default diff --git a/scripts/crd-run.py b/scripts/crd-run.py index daa7d6e..7a58c45 100644 --- a/scripts/crd-run.py +++ b/scripts/crd-run.py @@ -10,39 +10,41 @@ # Standard library # ----------------------- import argparse -from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait import glob import json -import os -import shutil -import time -import warnings -from typing import Any # ----------------------- # Logging # ----------------------- import logging +import os +import shutil +import time +import warnings +from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait +from typing import Any # ----------------------- # Third-party # ----------------------- import dask import dask.dataframe as dd -import pandas as pd -import numpy as np -from dask.distributed import Client, as_completed, performance_report, wait as dask_wait import lsdb +import numpy as np +import pandas as pd # ----------------------- # Project # ----------------------- from crossmatch_auto import crossmatch_auto from crossmatch_cross import crossmatch_tiebreak_safe +from dask.distributed import Client, as_completed, performance_report +from dask.distributed import wait as dask_wait from deduplication import ( REPRESENTATIVE_RADIUS_DIAGNOSTIC_COLUMN, count_global_edge_group_mismatches, count_global_tie_invariant_violations, + filter_dask_by_tie_treatment, filter_pandas_by_tie_treatment, run_dedup_with_lsdb_map_partitions, validate_spatial_safety, @@ -50,8 +52,7 @@ from executor import get_executor from product_handle import save_dataframe from resource_usage import ResourceUsageMonitor -from specz import prepare_catalog -from worker_health import WorkerFloorMonitor +from specz import prepare_catalog, validate_combine_configuration from utils import ( configure_exception_hook, configure_warning_handler, @@ -63,6 +64,7 @@ start_crc_log_collector, update_process_info, ) +from worker_health import WorkerFloorMonitor __all__ = ["main"] @@ -643,6 +645,16 @@ def main( ) log_init.info("START init: pipeline bootstrap") + cut_value = param_config.get("z_flag_homogenized_value_to_cut") + try: + cut_value_numeric = float(cut_value) if cut_value is not None else None + except (TypeError, ValueError): + cut_value_numeric = None + if cut_value_numeric == 0.0: + log_init.info( + "z_flag_homogenized cut disabled explicitly " + "(z_flag_homogenized_value_to_cut=0)." + ) configure_warning_handler(base_logger) warnings.filterwarnings( "ignore", @@ -761,9 +773,13 @@ def main( " - %s: %.1f MB", entry["internal_name"], _filesize_mb(entry["path"]) ) - combine_mode = param_config.get( - "combine_type", "concatenate_and_mark_duplicates" - ).lower() + combine_mode, validated_priorities = validate_combine_configuration( + param_config.get("combine_type", "concatenate_and_mark_duplicates"), + translation_config.get("tiebreaking_priority", []), + param_config.get("z_flag_homogenized_value_to_cut"), + log_init, + ) + translation_config["tiebreaking_priority"] = validated_priorities completed = read_completed_steps(os.path.join(temp_dir, "process_resume.log")) # --- Dask cluster/client --- @@ -1270,10 +1286,12 @@ def _submit_auto(executor: ThreadPoolExecutor, i: dict): df_final = dd.concat( [dd.read_parquet(i["prepared_path"]) for i in prepared_info] ) - if output_format == "hats": + if output_format in {"hats", "parquet", "csv"}: log_cross.info( - "Concatenate graph built lazily for HATS output " + "Concatenate graph kept lazy for distributed output " + "format=%s " "(npartitions=%s).", + output_format, getattr(df_final, "npartitions", "unknown"), ) else: @@ -1896,11 +1914,12 @@ def _submit_pair(executor: ThreadPoolExecutor) -> bool: # Final materialization: avoid pulling the full dataframe to the # driver when HATS output can consume a Dask dataframe directly. try: - if output_format == "hats": + if output_format in {"hats", "parquet", "csv"}: df_final = merged log_dedup.info( - "Keeping final merged dataframe lazy for HATS output " - "(npartitions=%s).", + "Keeping final merged dataframe lazy for distributed " + "output format=%s (npartitions=%s).", + output_format, getattr(df_final, "npartitions", "unknown"), ) else: @@ -1948,7 +1967,11 @@ def _submit_pair(executor: ThreadPoolExecutor) -> bool: "START consolidation: staging artifacts into process dir (base_dir=%s)", base_dir, ) - lazy_hats_output = output_format == "hats" and _is_dask_dataframe(df_final) + lazy_distributed_output = output_format in { + "hats", + "parquet", + "csv", + } and _is_dask_dataframe(df_final) log_cons.info( "Final dataframe backend=%s npartitions=%s columns=%d", @@ -1978,50 +2001,47 @@ def _submit_pair(executor: ThreadPoolExecutor) -> bool: ) tie_treatment_option = "remove_all" - if lazy_hats_output and tie_treatment_option == "draw_one": - log_cons.info( - "tie_treatment_option=draw_one requires pandas consolidation; " - "computing final dataframe before draw-one filtering." - ) - df_final = df_final.compute() - lazy_hats_output = False - - if lazy_hats_output: + if lazy_distributed_output: if "tie_result" not in df_final.columns: raise RuntimeError( "Expected 'tie_result' column for remove-duplicates mode" ) else: log_cons.info( - "Applying lazy Dask tie_result filter for HATS output " + "Applying lazy Dask tie_result filter for distributed output " "(tie_treatment_option=%s).", tie_treatment_option, ) try: - tie_num = ( - dd.to_numeric(df_final["tie_result"], errors="coerce") - .fillna(0) - .astype("int8") + requested_option = tie_treatment_option + df_final, tie_treatment_option = filter_dask_by_tie_treatment( + df_final, + tie_treatment_option, ) - if tie_treatment_option == "keep_all": - keep_mask = tie_num.isin([1, 2]) + if ( + requested_option == "draw_one" + and tie_treatment_option != "draw_one" + ): log_cons.info( - "Filtering rows lazily by tie_result in {1,2} (keep_all)." + "draw_one requires CRD_ID and group_id; falling back " + "to remove_all." ) - else: - keep_mask = tie_num.eq(1) + elif tie_treatment_option == "draw_one": log_cons.info( - "Filtering rows lazily by tie_result == 1 (%s).", - tie_treatment_option, + "Resolving hard ties lazily and deterministically by " + "the smallest CRD_ID in each group." ) - df_final = df_final.loc[keep_mask] + elif tie_treatment_option == "keep_all": + log_cons.info("Filtering lazily by tie_result in {1,2}.") + else: + log_cons.info("Filtering lazily by tie_result == 1.") except Exception as e: log_cons.error( "FAILED while applying lazy tie_result filter: %s", e ) raise - if lazy_hats_output: + if lazy_distributed_output: pass elif "tie_result" not in df_final.columns: raise RuntimeError( @@ -2054,7 +2074,7 @@ def _submit_pair(executor: ThreadPoolExecutor) -> bool: log_cons.error("FAILED while filtering by tie_treatment_option: %s", e) raise - if not lazy_hats_output: + if not lazy_distributed_output: _ensure_non_empty_final_dataframe( df_final, log_cons, @@ -2062,7 +2082,7 @@ def _submit_pair(executor: ThreadPoolExecutor) -> bool: ) else: log_cons.info( - "Deferring empty-output validation until after HATS parquet staging." + "Deferring empty-output validation until distributed parquet staging." ) # Stage final output with your save_dataframe diff --git a/setup.sh b/setup.sh deleted file mode 100755 index 0b8451e..0000000 --- a/setup.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -export APP_DIR=`pwd` - - -if [ -z "$DATASETS_DIR" ]; then - export DATASETS_DIR=$APP_DIR/data-example -fi - -echo "The directory containing the datasets is: " $DATASETS_DIR -read -p "Enter 'yes' to confirm the installation or 'no' to exit and reconfigure the variable DATASETS_DIR " dinput - -if [ "$dinput" != "yes" ]; then - echo Exiting... - exit 1 -fi - -pipe="${APP_DIR}/config.template.yaml" -DIRCONF=$(dirname "$pipe") -echo "$DIRCONF" -sed "s||$DATASETS_DIR|g" "$pipe" > "$DIRCONF/config.yaml" - -sed "s||$APP_DIR|g" $APP_DIR/env.template.sh > $APP_DIR/env.sh -sed -i'' -e "s||$DATASETS_DIR|g" $APP_DIR/env.sh diff --git a/test_configs/README.md b/test_configs/README.md deleted file mode 100644 index 38f458d..0000000 --- a/test_configs/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# WARNING! - -For these configuration files to work without requiring changes, the file to be used must be moved to the pipeline's root directory (```combine_redshift_dedup/```). - -If you don't move them, you'll need to change the paths in the catalogs and flags_translation.yaml in those configuration yaml files. \ No newline at end of file diff --git a/test_configs/config.all.yaml b/test_configs/config.all.yaml deleted file mode 100644 index fa903dc..0000000 --- a/test_configs/config.all.yaml +++ /dev/null @@ -1,254 +0,0 @@ -################################################################## -# Parameters file. Should follow the syntax defined in: -# http://docs.ansible.com/ansible/latest/YAMLSyntax.html -# More general reference: http://www.yaml.org/spec/1.2/spec.html - -executor: - name: "local" - args: - n_workers: 3 - threads_per_worker: 2 - memory_limit: "4GB" - -#executor: -# name: "slurm" -# args: -# instance: -# cores: 16 -# processes: 4 -# memory: "48GB" -# queue: "cpu_small" -# account: "hpc-public" -# job_extra_directives: -# - "--propagate" -# - "--time=2:00:00" -# scale: -# minimum_jobs: 4 -# maximum_jobs: 12 - -inputs: - specz: - # List of redshift catalogs to combine. Each entry contains: - # - path: full path to the file - # - internal_name: unique label used internally - # - columns: mapping from standard keys to column names in the file - - - path: "test_data/2dfgrs_final_release_random_sample.parquet" - internal_name: "001_2dfgrs_final_release" - columns: - id: "name" - ra: "ra_j2000_deg" - dec: "dec_j2000_deg" - z: "z" - z_flag: "quality" - z_err: null - survey: "survey_name" - - - path: "test_data/2dflens_final_release_random_sample.parquet" - internal_name: "002_2dflens_final_release" - columns: - id: "unique_id" - ra: "RA" - dec: "Dec" - z: "z" - z_flag: "qual" - z_err: null - survey: "survey_name" - - - path: "test_data/2mrs_v240_random_sample.parquet" - internal_name: "003_2mrs_v240" - columns: - id: "TMID" - ra: "RA" - dec: "DEC" - z: "redshift" - z_flag: null - z_err: "redshift_err" - survey: "survey_name" - - - path: "test_data/3dhst_v4.1.5_random_sample.parquet" - internal_name: "004_3dhst_v4.1.5" - columns: - id: "phot_id" - ra: "ra" - dec: "dec" - z: "z_best" - z_flag: null - z_err: null - survey: "survey_name" - - - path: "test_data/6dfgs_dr3_random_sample.parquet" - internal_name: "005_6dfgs_dr3" - columns: - id: "SPECID" - ra: "RA_deg" - dec: "DEC_deg" - z: "Z" - z_flag: "QUALITY" - z_err: null - survey: "survey_name" - - - path: "test_data/astrodeep_jwst_random_sample.parquet" - internal_name: "006_astrodeep_jwst" - columns: - id: "ID" - ra: "RA" - dec: "DEC" - z: "redshift" - z_flag: "flag" - z_err: null - survey: "survey_name" - - - path: "test_data/astrodeep-gs43_random_sample.parquet" - internal_name: "007_astrodeep" - columns: - id: "ID" - ra: "RA" - dec: "DEC" - z: "z_best" - z_flag: null - z_err: null - survey: "survey_name" - - - path: "test_data/desi_dr1_in_lsst_dp1_fields_random_sample.parquet" - internal_name: "008_desi_dr1" - columns: - id: "TARGETID" - ra: "TARGET_RA" - dec: "TARGET_DEC" - z: "Z" - z_flag: "ZWARN" - z_err: "ZERR" - survey: "survey_name" - - - path: "test_data/jades_dr3_random_sample.parquet" - internal_name: "009_jades_dr3" - columns: - id: "unique_id" - ra: "RA_TARG" - dec: "Dec_TARG" - z: "z_Spec" - z_flag: "z_Spec_flag" - z_err: null - survey: "survey_name" - - - path: "test_data/mosdef_final_release_random_sample.parquet" - internal_name: "010_mosdef_final_release" - columns: - id: "ID" - ra: "RA" - dec: "DEC" - z: "Z_MOSFIRE" - z_flag: "Z_QUAL" - z_err: null - survey: "survey_name" - - - path: "test_data/ozdes_dr2_random_sample.parquet" - internal_name: "011_ozdes_dr2" - columns: - id: "OzDES_ID" - ra: "Alpha_J2000" - dec: "Delta_J2000" - z: "z" - z_flag: "qop" - z_err: null - survey: "survey_name" - - - path: "test_data/primus_dr1_random_sample.parquet" - internal_name: "012_primus_dr1" - columns: - id: "OBJNAME" - ra: "RA" - dec: "DEC" - z: "Z" - z_flag: "ZQUALITY" - z_err: null - survey: "survey_name" - - - path: "test_data/vandels_dr4_random_sample.parquet" - internal_name: "013_vandels_dr4" - columns: - id: "id" - ra: "alpha" - dec: "delta" - z: "zspec" - z_flag: "zflg" - z_err: null - survey: "survey_name" - - - path: "test_data/vlt_vimos_v2.0.1_random_sample.parquet" - internal_name: "014_vlt_vimos_v2.0.1" - columns: - id: "ID_VIMOS" - ra: "RAJ2000_WFI" - dec: "DEJ2000_WFI" - z: "SPEC_Z" - z_flag: "campaign_qf" - z_err: null - survey: "survey_name" - - - path: "test_data/vuds_dr1_random_sample.parquet" - internal_name: "015_vuds_dr1" - columns: - id: "vuds_ident" - ra: "alpha" - dec: "delta" - z: "z_spec" - z_flag: "zflags" - z_err: null - survey: "survey_name" - - - path: "test_data/vvds_final_release_random_sample.parquet" - internal_name: "016_vvds_final_release" - columns: - id: "NUM" - ra: "ALPHA" - dec: "DELTA" - z: "Z" - z_flag: "ZFLAGS" - z_err: null - survey: "survey_name" - - - path: "test_data/z_cat_CANDELS_clean_sitcomtn-154_random_sample.parquet" - internal_name: "017_candels_sitcomtn" - columns: - id: null - ra: "RA" - dec: "DEC" - z: "redshift" - z_flag: "quality" - z_err: null - survey: "source" - - - path: "test_data/z_cat_NED_clean_sitcomtn-154_random_sample.parquet" - internal_name: "018_ned_sitcomtn" - columns: - id: null - ra: "RA" - dec: "DEC" - z: "redshift" - z_flag: "quality" - z_err: null - survey: "source" - - - path: "test_data/pipeline_generated_sample.parquet" - internal_name: "019_pipeline_sample" - columns: - id: "CRD_ID" - ra: "ra" - dec: "dec" - z: "z" - z_flag: "z_flag_homogenized" - z_err: "z_err" - survey: "survey" - -output_root_dir: "storage" -output_dir: "outputs" # Relative directory for final outputs -output_name: "crd" # Name prefix for the final output files -output_format: "parquet" # Output format: csv, parquet, hdf5, or fits - -param: - combine_type: "concatenate_and_mark_duplicates" # Options: "concatenate", "concatenate_and_mark_duplicates", or "concatenate_and_remove_duplicates" - tie_treatment_option: "remove_all" # Options (only for concatenate_and_remove_duplicates): "remove_all", "keep_all", "draw_one" - z_flag_homogenized_value_to_cut: 0.0 # Value to cut in the homogenized z_flag (cut >= value). 0.0 or null means keep all. - flags_translation_file: ../flags_translation.yaml # File with homogenization rules for z_flag and type diff --git a/test_configs/config.test.concatenate.yaml b/test_configs/config.test.concatenate.yaml deleted file mode 100644 index 8aa7da9..0000000 --- a/test_configs/config.test.concatenate.yaml +++ /dev/null @@ -1,89 +0,0 @@ -################################################################## -# Parameters file. Should follow the syntax defined in: -# http://docs.ansible.com/ansible/latest/YAMLSyntax.html -# More general reference: http://www.yaml.org/spec/1.2/spec.html - -executor: - name: "local" - args: - n_workers: 3 - threads_per_worker: 2 - memory_limit: "4GB" - -#executor: -# name: "slurm" -# args: -# instance: -# cores: 16 -# processes: 4 -# memory: "48GB" -# queue: "cpu_small" -# account: "hpc-public" -# job_extra_directives: -# - "--propagate" -# - "--time=12:00:00" -# scale: -# minimum_jobs: 4 -# maximum_jobs: 12 - -inputs: - specz: - # List of redshift catalogs to combine. Each entry contains: - # - path: full path to the file - # - internal_name: unique label used internally - # - columns: mapping from standard keys to column names in the file - - - path: "test_data/2dfgrs_final_release_random_sample.parquet" - internal_name: "001_2dfgrs_final_release" - columns: - id: "name" - ra: "ra_j2000_deg" - dec: "dec_j2000_deg" - z: "z" - z_flag: "quality" - z_err: null - survey: "survey_name" - - - path: "test_data/2dflens_final_release_random_sample.parquet" - internal_name: "002_2dflens_final_release" - columns: - id: "unique_id" - ra: "RA" - dec: "Dec" - z: "z" - z_flag: "qual" - z_err: null - survey: "survey_name" - - - path: "test_data/2mrs_v240_random_sample.parquet" - internal_name: "003_2mrs_v240" - columns: - id: "TMID" - ra: "RA" - dec: "DEC" - z: "redshift" - z_flag: null - z_err: "redshift_err" - survey: "survey_name" - - - path: 'test_data/z_cat_NED_clean_sitcomtn-154_random_sample.parquet' - internal_name: "004_ned_sitcomtn" - columns: - id: null - ra: "RA" - dec: "DEC" - z: "redshift" - z_flag: "quality" - z_err: null - survey: "source" - -output_root_dir: "storage" -output_dir: "outputs" # Relative directory for final outputs -output_name: "crd" # Name prefix for the final output files -output_format: "parquet" # Output format: csv, parquet, hdf5, or fits - -param: - combine_type: "concatenate" # Options: "concatenate", "concatenate_and_mark_duplicates", or "concatenate_and_remove_duplicates" - tie_treatment_option: "remove_all" # Options (only for concatenate_and_remove_duplicates): "remove_all", "keep_all", "draw_one" - z_flag_homogenized_value_to_cut: 0.0 # Value to cut in the homogenized z_flag (cut >= value). 0.0 or null means keep all. - flags_translation_file: ../flags_translation.yaml # File with homogenization rules for z_flag and type diff --git a/test_configs/config.test.elais.yaml b/test_configs/config.test.elais.yaml deleted file mode 100644 index c5335d6..0000000 --- a/test_configs/config.test.elais.yaml +++ /dev/null @@ -1,42 +0,0 @@ -executor: - name: "local" - args: - n_workers: 3 - threads_per_worker: 2 - memory_limit: "4GB" - -inputs: - specz: - - path: "./test_data/elaisfbmc.parquet" - internal_name: "001_elaisfbmc" - columns: - id: "ELAIS" - ra: "RAdeg" - dec: "DEdeg" - z: "zbest" - z_flag: "zbest_quality" - z_err: null - survey: "survey_name" - - - path: "./test_data/elaiss1oid.parquet" - internal_name: "002_elaiss1oid" - columns: - id: "XMMES1" - ra: "RAdeg" - dec: "DEdeg" - z: "zfinal" - z_flag: "zfinal_quality" - z_err: null - survey: "survey_name" - -output_root_dir: "storage" -output_dir: "outputs" -output_name: "crd" -output_format: "parquet" - -param: - combine_type: "concatenate_and_mark_duplicates" - z_flag_homogenized_value_to_cut: 0.0 - insert_DP1_footprint_flag: false - insert_rubin_footprint_flag: true - flags_translation_file: ./flags_translation.yaml diff --git a/test_configs/config.test.homogenized_columns_out_of_range.yaml b/test_configs/config.test.homogenized_columns_out_of_range.yaml deleted file mode 100644 index 2922cd2..0000000 --- a/test_configs/config.test.homogenized_columns_out_of_range.yaml +++ /dev/null @@ -1,79 +0,0 @@ -################################################################## -# Parameters file. Should follow the syntax defined in: -# http://docs.ansible.com/ansible/latest/YAMLSyntax.html -# More general reference: http://www.yaml.org/spec/1.2/spec.html - -executor: - name: "local" - args: - n_workers: 3 - threads_per_worker: 2 - memory_limit: "4GB" - -#executor: -# name: "slurm" -# args: -# instance: -# cores: 16 -# processes: 4 -# memory: "48GB" -# queue: "cpu_small" -# account: "hpc-public" -# job_extra_directives: -# - "--propagate" -# - "--time=12:00:00" -# scale: -# minimum_jobs: 4 -# maximum_jobs: 12 - -inputs: - specz: - # List of redshift catalogs to combine. Each entry contains: - # - path: full path to the file - # - internal_name: unique label used internally - # - columns: mapping from standard keys to column names in the file - - - path: "test_data/2dfgrs_final_release_random_sample.parquet" - internal_name: "001_2dfgrs_final_release" - columns: - id: "name" - ra: "ra_j2000_deg" - dec: "dec_j2000_deg" - z: "z" - z_flag: "quality" - z_err: null - survey: "survey_name" - - - path: "test_data/2dflens_final_release_random_sample.parquet" - internal_name: "002_2dflens_final_release" - columns: - id: "unique_id" - ra: "RA" - dec: "Dec" - z: "z" - z_flag: "qual" - z_err: null - survey: "survey_name" - - - path: "test_data/test_homogenized_columns_out_of_range.parquet" - internal_name: "003_homogenized_out_of_range" - columns: - id: "id" - ra: "ra" - dec: "dec" - z: "z" - z_flag: "z_flag_homogenized" - z_err: null - survey: "survey" - - -output_root_dir: "storage" -output_dir: "outputs" # Relative directory for final outputs -output_name: "crd" # Name prefix for the final output files -output_format: "parquet" # Output format: csv, parquet, hdf5, or fits - -param: - combine_type: "concatenate_and_mark_duplicates" # Options: "concatenate", "concatenate_and_mark_duplicates", or "concatenate_and_remove_duplicates" - tie_treatment_option: "remove_all" # Options (only for concatenate_and_remove_duplicates): "remove_all", "keep_all", "draw_one" - z_flag_homogenized_value_to_cut: 0.0 # Value to cut in the homogenized z_flag (cut >= value). 0.0 or null means keep all. - flags_translation_file: ../flags_translation.yaml # File with homogenization rules for z_flag and type diff --git a/test_configs/config.test.mark.yaml b/test_configs/config.test.mark.yaml deleted file mode 100644 index e2846fe..0000000 --- a/test_configs/config.test.mark.yaml +++ /dev/null @@ -1,89 +0,0 @@ -################################################################## -# Parameters file. Should follow the syntax defined in: -# http://docs.ansible.com/ansible/latest/YAMLSyntax.html -# More general reference: http://www.yaml.org/spec/1.2/spec.html - -executor: - name: "local" - args: - n_workers: 3 - threads_per_worker: 2 - memory_limit: "4GB" - -#executor: -# name: "slurm" -# args: -# instance: -# cores: 16 -# processes: 4 -# memory: "48GB" -# queue: "cpu_small" -# account: "hpc-public" -# job_extra_directives: -# - "--propagate" -# - "--time=12:00:00" -# scale: -# minimum_jobs: 4 -# maximum_jobs: 12 - -inputs: - specz: - # List of redshift catalogs to combine. Each entry contains: - # - path: full path to the file - # - internal_name: unique label used internally - # - columns: mapping from standard keys to column names in the file - - - path: "test_data/2dfgrs_final_release_random_sample.parquet" - internal_name: "001_2dfgrs_final_release" - columns: - id: "name" - ra: "ra_j2000_deg" - dec: "dec_j2000_deg" - z: "z" - z_flag: "quality" - z_err: null - survey: "survey_name" - - - path: "test_data/2dflens_final_release_random_sample.parquet" - internal_name: "002_2dflens_final_release" - columns: - id: "unique_id" - ra: "RA" - dec: "Dec" - z: "z" - z_flag: "qual" - z_err: null - survey: "survey_name" - - - path: "test_data/2mrs_v240_random_sample.parquet" - internal_name: "003_2mrs_v240" - columns: - id: "TMID" - ra: "RA" - dec: "DEC" - z: "redshift" - z_flag: null - z_err: "redshift_err" - survey: "survey_name" - - - path: 'test_data/z_cat_NED_clean_sitcomtn-154_random_sample.parquet' - internal_name: "004_ned_sitcomtn" - columns: - id: null - ra: "RA" - dec: "DEC" - z: "redshift" - z_flag: "quality" - z_err: null - survey: "source" - -output_root_dir: "storage" -output_dir: "outputs" # Relative directory for final outputs -output_name: "crd" # Name prefix for the final output files -output_format: "parquet" # Output format: csv, parquet, hdf5, or fits - -param: - combine_type: "concatenate_and_mark_duplicates" # Options: "concatenate", "concatenate_and_mark_duplicates", or "concatenate_and_remove_duplicates" - tie_treatment_option: "remove_all" # Options (only for concatenate_and_remove_duplicates): "remove_all", "keep_all", "draw_one" - z_flag_homogenized_value_to_cut: 0.0 # Value to cut in the homogenized z_flag (cut >= value). 0.0 or null means keep all. - flags_translation_file: ../flags_translation.yaml # File with homogenization rules for z_flag and type diff --git a/test_configs/config.test.remove.yaml b/test_configs/config.test.remove.yaml deleted file mode 100644 index 4964120..0000000 --- a/test_configs/config.test.remove.yaml +++ /dev/null @@ -1,89 +0,0 @@ -################################################################## -# Parameters file. Should follow the syntax defined in: -# http://docs.ansible.com/ansible/latest/YAMLSyntax.html -# More general reference: http://www.yaml.org/spec/1.2/spec.html - -executor: - name: "local" - args: - n_workers: 3 - threads_per_worker: 2 - memory_limit: "4GB" - -#executor: -# name: "slurm" -# args: -# instance: -# cores: 16 -# processes: 4 -# memory: "48GB" -# queue: "cpu_small" -# account: "hpc-public" -# job_extra_directives: -# - "--propagate" -# - "--time=12:00:00" -# scale: -# minimum_jobs: 4 -# maximum_jobs: 12 - -inputs: - specz: - # List of redshift catalogs to combine. Each entry contains: - # - path: full path to the file - # - internal_name: unique label used internally - # - columns: mapping from standard keys to column names in the file - - - path: "test_data/2dfgrs_final_release_random_sample.parquet" - internal_name: "001_2dfgrs_final_release" - columns: - id: "name" - ra: "ra_j2000_deg" - dec: "dec_j2000_deg" - z: "z" - z_flag: "quality" - z_err: null - survey: "survey_name" - - - path: "test_data/2dflens_final_release_random_sample.parquet" - internal_name: "002_2dflens_final_release" - columns: - id: "unique_id" - ra: "RA" - dec: "Dec" - z: "z" - z_flag: "qual" - z_err: null - survey: "survey_name" - - - path: "test_data/2mrs_v240_random_sample.parquet" - internal_name: "003_2mrs_v240" - columns: - id: "TMID" - ra: "RA" - dec: "DEC" - z: "redshift" - z_flag: null - z_err: "redshift_err" - survey: "survey_name" - - - path: 'test_data/z_cat_NED_clean_sitcomtn-154_random_sample.parquet' - internal_name: "004_ned_sitcomtn" - columns: - id: null - ra: "RA" - dec: "DEC" - z: "redshift" - z_flag: "quality" - z_err: null - survey: "source" - -output_root_dir: "storage" -output_dir: "outputs" # Relative directory for final outputs -output_name: "crd" # Name prefix for the final output files -output_format: "parquet" # Output format: csv, parquet, hdf5, or fits - -param: - combine_type: "concatenate_and_remove_duplicates" # Options: "concatenate", "concatenate_and_mark_duplicates", or "concatenate_and_remove_duplicates" - tie_treatment_option: "remove_all" # Options (only for concatenate_and_remove_duplicates): "remove_all", "keep_all", "draw_one" - z_flag_homogenized_value_to_cut: 0.0 # Value to cut in the homogenized z_flag (cut >= value). 0.0 or null means keep all. - flags_translation_file: ../flags_translation.yaml # File with homogenization rules for z_flag and type diff --git a/test_configs/config.test.survey_col_missing.yaml b/test_configs/config.test.survey_col_missing.yaml deleted file mode 100644 index 931618a..0000000 --- a/test_configs/config.test.survey_col_missing.yaml +++ /dev/null @@ -1,79 +0,0 @@ -################################################################## -# Parameters file. Should follow the syntax defined in: -# http://docs.ansible.com/ansible/latest/YAMLSyntax.html -# More general reference: http://www.yaml.org/spec/1.2/spec.html - -executor: - name: "local" - args: - n_workers: 3 - threads_per_worker: 2 - memory_limit: "4GB" - -#executor: -# name: "slurm" -# args: -# instance: -# cores: 16 -# processes: 4 -# memory: "48GB" -# queue: "cpu_small" -# account: "hpc-public" -# job_extra_directives: -# - "--propagate" -# - "--time=12:00:00" -# scale: -# minimum_jobs: 4 -# maximum_jobs: 12 - -inputs: - specz: - # List of redshift catalogs to combine. Each entry contains: - # - path: full path to the file - # - internal_name: unique label used internally - # - columns: mapping from standard keys to column names in the file - - - path: "test_data/2dfgrs_final_release_random_sample.parquet" - internal_name: "001_2dfgrs_final_release" - columns: - id: "name" - ra: "ra_j2000_deg" - dec: "dec_j2000_deg" - z: "z" - z_flag: "quality" - z_err: null - survey: "survey_name" - - - path: "test_data/2dflens_final_release_random_sample.parquet" - internal_name: "002_2dflens_final_release" - columns: - id: "unique_id" - ra: "RA" - dec: "Dec" - z: "z" - z_flag: "qual" - z_err: null - survey: "survey_name" - - - path: "test_data/test_survey_col_missing.parquet" - internal_name: "003_survey_col_missing" - columns: - id: "id" - ra: "ra" - dec: "dec" - z: "z" - z_flag: "z_flag" - z_err: null - survey: null - - -output_root_dir: "storage" -output_dir: "outputs" # Relative directory for final outputs -output_name: "crd" # Name prefix for the final output files -output_format: "parquet" # Output format: csv, parquet, hdf5, or fits - -param: - combine_type: "concatenate_and_mark_duplicates" # Options: "concatenate", "concatenate_and_mark_duplicates", or "concatenate_and_remove_duplicates" - tie_treatment_option: "remove_all" # Options (only for concatenate_and_remove_duplicates): "remove_all", "keep_all", "draw_one" - z_flag_homogenized_value_to_cut: 0.0 # Value to cut in the homogenized z_flag (cut >= value). 0.0 or null means keep all. - flags_translation_file: ../flags_translation.yaml # File with homogenization rules for z_flag and type diff --git a/test_configs/config.test.survey_not_supported.yaml b/test_configs/config.test.survey_not_supported.yaml deleted file mode 100644 index c6e93e4..0000000 --- a/test_configs/config.test.survey_not_supported.yaml +++ /dev/null @@ -1,79 +0,0 @@ -################################################################## -# Parameters file. Should follow the syntax defined in: -# http://docs.ansible.com/ansible/latest/YAMLSyntax.html -# More general reference: http://www.yaml.org/spec/1.2/spec.html - -executor: - name: "local" - args: - n_workers: 3 - threads_per_worker: 2 - memory_limit: "4GB" - -#executor: -# name: "slurm" -# args: -# instance: -# cores: 16 -# processes: 4 -# memory: "48GB" -# queue: "cpu_small" -# account: "hpc-public" -# job_extra_directives: -# - "--propagate" -# - "--time=12:00:00" -# scale: -# minimum_jobs: 4 -# maximum_jobs: 12 - -inputs: - specz: - # List of redshift catalogs to combine. Each entry contains: - # - path: full path to the file - # - internal_name: unique label used internally - # - columns: mapping from standard keys to column names in the file - - - path: "test_data/2dfgrs_final_release_random_sample.parquet" - internal_name: "001_2dfgrs_final_release" - columns: - id: "name" - ra: "ra_j2000_deg" - dec: "dec_j2000_deg" - z: "z" - z_flag: "quality" - z_err: null - survey: "survey_name" - - - path: "test_data/2dflens_final_release_random_sample.parquet" - internal_name: "002_2dflens_final_release" - columns: - id: "unique_id" - ra: "RA" - dec: "Dec" - z: "z" - z_flag: "qual" - z_err: null - survey: "survey_name" - - - path: "test_data/test_survey_not_supported.parquet" - internal_name: "003_survey_not_supported" - columns: - id: "id" - ra: "ra" - dec: "dec" - z: "z" - z_flag: "z_flag" - z_err: null - survey: "survey" - - -output_root_dir: "storage" -output_dir: "outputs" # Relative directory for final outputs -output_name: "crd" # Name prefix for the final output files -output_format: "parquet" # Output format: csv, parquet, hdf5, or fits - -param: - combine_type: "concatenate_and_mark_duplicates" # Options: "concatenate", "concatenate_and_mark_duplicates", or "concatenate_and_remove_duplicates" - tie_treatment_option: "remove_all" # Options (only for concatenate_and_remove_duplicates): "remove_all", "keep_all", "draw_one" - z_flag_homogenized_value_to_cut: 0.0 # Value to cut in the homogenized z_flag (cut >= value). 0.0 or null means keep all. - flags_translation_file: ../flags_translation.yaml # File with homogenization rules for z_flag and type diff --git a/test_configs/config.test.survey_supported_but_wrong_flags.yaml b/test_configs/config.test.survey_supported_but_wrong_flags.yaml deleted file mode 100644 index 881f891..0000000 --- a/test_configs/config.test.survey_supported_but_wrong_flags.yaml +++ /dev/null @@ -1,79 +0,0 @@ -################################################################## -# Parameters file. Should follow the syntax defined in: -# http://docs.ansible.com/ansible/latest/YAMLSyntax.html -# More general reference: http://www.yaml.org/spec/1.2/spec.html - -executor: - name: "local" - args: - n_workers: 3 - threads_per_worker: 2 - memory_limit: "4GB" - -#executor: -# name: "slurm" -# args: -# instance: -# cores: 16 -# processes: 4 -# memory: "48GB" -# queue: "cpu_small" -# account: "hpc-public" -# job_extra_directives: -# - "--propagate" -# - "--time=12:00:00" -# scale: -# minimum_jobs: 4 -# maximum_jobs: 12 - -inputs: - specz: - # List of redshift catalogs to combine. Each entry contains: - # - path: full path to the file - # - internal_name: unique label used internally - # - columns: mapping from standard keys to column names in the file - - - path: "test_data/2dfgrs_final_release_random_sample.parquet" - internal_name: "001_2dfgrs_final_release" - columns: - id: "name" - ra: "ra_j2000_deg" - dec: "dec_j2000_deg" - z: "z" - z_flag: "quality" - z_err: null - survey: "survey_name" - - - path: "test_data/2dflens_final_release_random_sample.parquet" - internal_name: "002_2dflens_final_release" - columns: - id: "unique_id" - ra: "RA" - dec: "Dec" - z: "z" - z_flag: "qual" - z_err: null - survey: "survey_name" - - - path: "test_data/test_survey_supported_but_wrong_flags.parquet" - internal_name: "003_survey_supported_but_wrong_flags" - columns: - id: "id" - ra: "ra" - dec: "dec" - z: "z" - z_flag: "z_flag" - z_err: null - survey: "survey" - - -output_root_dir: "storage" -output_dir: "outputs" # Relative directory for final outputs -output_name: "crd" # Name prefix for the final output files -output_format: "parquet" # Output format: csv, parquet, hdf5, or fits - -param: - combine_type: "concatenate_and_mark_duplicates" # Options: "concatenate", "concatenate_and_mark_duplicates", or "concatenate_and_remove_duplicates" - tie_treatment_option: "remove_all" # Options (only for concatenate_and_remove_duplicates): "remove_all", "keep_all", "draw_one" - z_flag_homogenized_value_to_cut: 0.0 # Value to cut in the homogenized z_flag (cut >= value). 0.0 or null means keep all. - flags_translation_file: ../flags_translation.yaml # File with homogenization rules for z_flag and type diff --git a/test_data/3dhst_v4.1.5_random_sample.parquet b/test_data/3dhst_v4.1.5_random_sample.parquet deleted file mode 100644 index 7bdb13a..0000000 Binary files a/test_data/3dhst_v4.1.5_random_sample.parquet and /dev/null differ diff --git a/test_data/6dfgs_dr3_random_sample.parquet b/test_data/6dfgs_dr3_random_sample.parquet deleted file mode 100644 index 0cb7ac2..0000000 Binary files a/test_data/6dfgs_dr3_random_sample.parquet and /dev/null differ diff --git a/test_data/astrodeep-gs43_random_sample.parquet b/test_data/astrodeep-gs43_random_sample.parquet deleted file mode 100644 index 464878a..0000000 Binary files a/test_data/astrodeep-gs43_random_sample.parquet and /dev/null differ diff --git a/test_data/astrodeep_jwst_random_sample.parquet b/test_data/astrodeep_jwst_random_sample.parquet deleted file mode 100644 index a779c9d..0000000 Binary files a/test_data/astrodeep_jwst_random_sample.parquet and /dev/null differ diff --git a/test_data/desi_dr1_in_lsst_dp1_fields_random_sample.parquet b/test_data/desi_dr1_in_lsst_dp1_fields_random_sample.parquet deleted file mode 100644 index dc6fa08..0000000 Binary files a/test_data/desi_dr1_in_lsst_dp1_fields_random_sample.parquet and /dev/null differ diff --git a/test_data/elaisfbmc.parquet b/test_data/elaisfbmc.parquet deleted file mode 100644 index 984fa9b..0000000 Binary files a/test_data/elaisfbmc.parquet and /dev/null differ diff --git a/test_data/elaisfbmc_collection/catalog/dataset/Norder=0/Dir=0/Npix=2.parquet b/test_data/elaisfbmc_collection/catalog/dataset/Norder=0/Dir=0/Npix=2.parquet deleted file mode 100644 index 4297eec..0000000 Binary files a/test_data/elaisfbmc_collection/catalog/dataset/Norder=0/Dir=0/Npix=2.parquet and /dev/null differ diff --git a/test_data/elaisfbmc_collection/catalog/dataset/Norder=0/Dir=0/Npix=8.parquet b/test_data/elaisfbmc_collection/catalog/dataset/Norder=0/Dir=0/Npix=8.parquet deleted file mode 100644 index 875606d..0000000 Binary files a/test_data/elaisfbmc_collection/catalog/dataset/Norder=0/Dir=0/Npix=8.parquet and /dev/null differ diff --git a/test_data/elaisfbmc_collection/catalog/dataset/_common_metadata b/test_data/elaisfbmc_collection/catalog/dataset/_common_metadata deleted file mode 100644 index 083308d..0000000 Binary files a/test_data/elaisfbmc_collection/catalog/dataset/_common_metadata and /dev/null differ diff --git a/test_data/elaisfbmc_collection/catalog/dataset/_metadata b/test_data/elaisfbmc_collection/catalog/dataset/_metadata deleted file mode 100644 index 1c373b2..0000000 Binary files a/test_data/elaisfbmc_collection/catalog/dataset/_metadata and /dev/null differ diff --git a/test_data/elaisfbmc_collection/catalog/hats.properties b/test_data/elaisfbmc_collection/catalog/hats.properties deleted file mode 100644 index 8242230..0000000 --- a/test_data/elaisfbmc_collection/catalog/hats.properties +++ /dev/null @@ -1,19 +0,0 @@ -#HATS catalog -obs_collection=catalog -dataproduct_type=object -hats_nrows=3762 -hats_col_ra=RAdeg -hats_col_dec=DEdeg -hats_col_healpix=_healpix_29 -hats_col_healpix_order=29 -hats_npix_suffix=.parquet -hats_skymap_order=10 -hats_max_rows=1000000 -moc_sky_fraction=0.16667 -hats_builder=hats-import v0.9.0, hats v0.9.0 -hats_creation_date=2026-06-01T19:08UTC -hats_estsize=503 -hats_release_date=2025-08-22 -hats_version=v1.0 -hats_cols_sort=ELAIS -hats_order=0 diff --git a/test_data/elaisfbmc_collection/catalog/partition_info.csv b/test_data/elaisfbmc_collection/catalog/partition_info.csv deleted file mode 100644 index eeec535..0000000 --- a/test_data/elaisfbmc_collection/catalog/partition_info.csv +++ /dev/null @@ -1,3 +0,0 @@ -Norder,Npix -0,2 -0,8 diff --git a/test_data/elaisfbmc_collection/catalog/point_map.fits b/test_data/elaisfbmc_collection/catalog/point_map.fits deleted file mode 100644 index 258c596..0000000 Binary files a/test_data/elaisfbmc_collection/catalog/point_map.fits and /dev/null differ diff --git a/test_data/elaisfbmc_collection/catalog/properties b/test_data/elaisfbmc_collection/catalog/properties deleted file mode 100644 index 8242230..0000000 --- a/test_data/elaisfbmc_collection/catalog/properties +++ /dev/null @@ -1,19 +0,0 @@ -#HATS catalog -obs_collection=catalog -dataproduct_type=object -hats_nrows=3762 -hats_col_ra=RAdeg -hats_col_dec=DEdeg -hats_col_healpix=_healpix_29 -hats_col_healpix_order=29 -hats_npix_suffix=.parquet -hats_skymap_order=10 -hats_max_rows=1000000 -moc_sky_fraction=0.16667 -hats_builder=hats-import v0.9.0, hats v0.9.0 -hats_creation_date=2026-06-01T19:08UTC -hats_estsize=503 -hats_release_date=2025-08-22 -hats_version=v1.0 -hats_cols_sort=ELAIS -hats_order=0 diff --git a/test_data/elaisfbmc_collection/catalog/skymap.fits b/test_data/elaisfbmc_collection/catalog/skymap.fits deleted file mode 100644 index 258c596..0000000 Binary files a/test_data/elaisfbmc_collection/catalog/skymap.fits and /dev/null differ diff --git a/test_data/elaisfbmc_collection/collection.properties b/test_data/elaisfbmc_collection/collection.properties deleted file mode 100644 index 90ab919..0000000 --- a/test_data/elaisfbmc_collection/collection.properties +++ /dev/null @@ -1,9 +0,0 @@ -#HATS Collection -obs_collection=elaisfbmc_collection -hats_primary_table_url=catalog -all_margins=margin_10arcs -hats_builder=hats-import v0.9.0, hats v0.9.0 -hats_creation_date=2026-06-01T19:08UTC -hats_estsize=519 -hats_release_date=2025-08-22 -hats_version=v1.0 diff --git a/test_data/elaisfbmc_collection/margin_10arcs/dataset/_common_metadata b/test_data/elaisfbmc_collection/margin_10arcs/dataset/_common_metadata deleted file mode 100644 index 083308d..0000000 Binary files a/test_data/elaisfbmc_collection/margin_10arcs/dataset/_common_metadata and /dev/null differ diff --git a/test_data/elaisfbmc_collection/margin_10arcs/dataset/_metadata b/test_data/elaisfbmc_collection/margin_10arcs/dataset/_metadata deleted file mode 100644 index 083308d..0000000 Binary files a/test_data/elaisfbmc_collection/margin_10arcs/dataset/_metadata and /dev/null differ diff --git a/test_data/elaisfbmc_collection/margin_10arcs/hats.properties b/test_data/elaisfbmc_collection/margin_10arcs/hats.properties deleted file mode 100644 index 1c9e394..0000000 --- a/test_data/elaisfbmc_collection/margin_10arcs/hats.properties +++ /dev/null @@ -1,18 +0,0 @@ -#HATS catalog -obs_collection=margin_10arcs -dataproduct_type=margin -hats_nrows=0 -hats_col_ra=RAdeg -hats_col_dec=DEdeg -hats_col_healpix=_healpix_29 -hats_col_healpix_order=29 -hats_primary_table_url=/home/luigi/linea/pzserver_combine_redshift_dedup/test_data/elaisfbmc_collection/catalog -hats_margin_threshold=10.0 -hats_npix_suffix=.parquet -moc_sky_fraction=0.0 -hats_order=0 -hats_builder=hats-import v0.9.0, hats v0.9.0 -hats_creation_date=2026-06-01T19:08UTC -hats_estsize=14 -hats_release_date=2025-08-22 -hats_version=v1.0 diff --git a/test_data/elaisfbmc_collection/margin_10arcs/partition_info.csv b/test_data/elaisfbmc_collection/margin_10arcs/partition_info.csv deleted file mode 100644 index 7117bdc..0000000 --- a/test_data/elaisfbmc_collection/margin_10arcs/partition_info.csv +++ /dev/null @@ -1 +0,0 @@ -Norder,Npix diff --git a/test_data/elaisfbmc_collection/margin_10arcs/properties b/test_data/elaisfbmc_collection/margin_10arcs/properties deleted file mode 100644 index 1c9e394..0000000 --- a/test_data/elaisfbmc_collection/margin_10arcs/properties +++ /dev/null @@ -1,18 +0,0 @@ -#HATS catalog -obs_collection=margin_10arcs -dataproduct_type=margin -hats_nrows=0 -hats_col_ra=RAdeg -hats_col_dec=DEdeg -hats_col_healpix=_healpix_29 -hats_col_healpix_order=29 -hats_primary_table_url=/home/luigi/linea/pzserver_combine_redshift_dedup/test_data/elaisfbmc_collection/catalog -hats_margin_threshold=10.0 -hats_npix_suffix=.parquet -moc_sky_fraction=0.0 -hats_order=0 -hats_builder=hats-import v0.9.0, hats v0.9.0 -hats_creation_date=2026-06-01T19:08UTC -hats_estsize=14 -hats_release_date=2025-08-22 -hats_version=v1.0 diff --git a/test_data/elaiss1oid.parquet b/test_data/elaiss1oid.parquet deleted file mode 100644 index 78992f7..0000000 Binary files a/test_data/elaiss1oid.parquet and /dev/null differ diff --git a/test_data/elaiss1oid_collection/catalog/dataset/Norder=2/Dir=0/Npix=138.parquet b/test_data/elaiss1oid_collection/catalog/dataset/Norder=2/Dir=0/Npix=138.parquet deleted file mode 100644 index f622568..0000000 Binary files a/test_data/elaiss1oid_collection/catalog/dataset/Norder=2/Dir=0/Npix=138.parquet and /dev/null differ diff --git a/test_data/elaiss1oid_collection/catalog/dataset/_common_metadata b/test_data/elaiss1oid_collection/catalog/dataset/_common_metadata deleted file mode 100644 index 68381f7..0000000 Binary files a/test_data/elaiss1oid_collection/catalog/dataset/_common_metadata and /dev/null differ diff --git a/test_data/elaiss1oid_collection/catalog/dataset/_metadata b/test_data/elaiss1oid_collection/catalog/dataset/_metadata deleted file mode 100644 index 2475415..0000000 Binary files a/test_data/elaiss1oid_collection/catalog/dataset/_metadata and /dev/null differ diff --git a/test_data/elaiss1oid_collection/catalog/hats.properties b/test_data/elaiss1oid_collection/catalog/hats.properties deleted file mode 100644 index 7c902f1..0000000 --- a/test_data/elaiss1oid_collection/catalog/hats.properties +++ /dev/null @@ -1,19 +0,0 @@ -#HATS catalog -obs_collection=catalog -dataproduct_type=object -hats_nrows=478 -hats_col_ra=RAdeg -hats_col_dec=DEdeg -hats_col_healpix=_healpix_29 -hats_col_healpix_order=29 -hats_npix_suffix=.parquet -hats_skymap_order=10 -hats_max_rows=1000000 -moc_sky_fraction=0.00521 -hats_builder=hats-import v0.9.0, hats v0.9.0 -hats_creation_date=2026-06-01T19:10UTC -hats_estsize=178 -hats_release_date=2025-08-22 -hats_version=v1.0 -hats_cols_sort=XMMES1 -hats_order=2 diff --git a/test_data/elaiss1oid_collection/catalog/partition_info.csv b/test_data/elaiss1oid_collection/catalog/partition_info.csv deleted file mode 100644 index ae2031f..0000000 --- a/test_data/elaiss1oid_collection/catalog/partition_info.csv +++ /dev/null @@ -1,2 +0,0 @@ -Norder,Npix -2,138 diff --git a/test_data/elaiss1oid_collection/catalog/point_map.fits b/test_data/elaiss1oid_collection/catalog/point_map.fits deleted file mode 100644 index 6ca123b..0000000 Binary files a/test_data/elaiss1oid_collection/catalog/point_map.fits and /dev/null differ diff --git a/test_data/elaiss1oid_collection/catalog/properties b/test_data/elaiss1oid_collection/catalog/properties deleted file mode 100644 index 7c902f1..0000000 --- a/test_data/elaiss1oid_collection/catalog/properties +++ /dev/null @@ -1,19 +0,0 @@ -#HATS catalog -obs_collection=catalog -dataproduct_type=object -hats_nrows=478 -hats_col_ra=RAdeg -hats_col_dec=DEdeg -hats_col_healpix=_healpix_29 -hats_col_healpix_order=29 -hats_npix_suffix=.parquet -hats_skymap_order=10 -hats_max_rows=1000000 -moc_sky_fraction=0.00521 -hats_builder=hats-import v0.9.0, hats v0.9.0 -hats_creation_date=2026-06-01T19:10UTC -hats_estsize=178 -hats_release_date=2025-08-22 -hats_version=v1.0 -hats_cols_sort=XMMES1 -hats_order=2 diff --git a/test_data/elaiss1oid_collection/catalog/skymap.fits b/test_data/elaiss1oid_collection/catalog/skymap.fits deleted file mode 100644 index 6ca123b..0000000 Binary files a/test_data/elaiss1oid_collection/catalog/skymap.fits and /dev/null differ diff --git a/test_data/elaiss1oid_collection/collection.properties b/test_data/elaiss1oid_collection/collection.properties deleted file mode 100644 index 2c04e2c..0000000 --- a/test_data/elaiss1oid_collection/collection.properties +++ /dev/null @@ -1,9 +0,0 @@ -#HATS Collection -obs_collection=elaiss1oid_collection -hats_primary_table_url=catalog -all_margins=margin_10arcs -hats_builder=hats-import v0.9.0, hats v0.9.0 -hats_creation_date=2026-06-01T19:10UTC -hats_estsize=193 -hats_release_date=2025-08-22 -hats_version=v1.0 diff --git a/test_data/elaiss1oid_collection/margin_10arcs/dataset/_common_metadata b/test_data/elaiss1oid_collection/margin_10arcs/dataset/_common_metadata deleted file mode 100644 index 68381f7..0000000 Binary files a/test_data/elaiss1oid_collection/margin_10arcs/dataset/_common_metadata and /dev/null differ diff --git a/test_data/elaiss1oid_collection/margin_10arcs/dataset/_metadata b/test_data/elaiss1oid_collection/margin_10arcs/dataset/_metadata deleted file mode 100644 index 68381f7..0000000 Binary files a/test_data/elaiss1oid_collection/margin_10arcs/dataset/_metadata and /dev/null differ diff --git a/test_data/elaiss1oid_collection/margin_10arcs/hats.properties b/test_data/elaiss1oid_collection/margin_10arcs/hats.properties deleted file mode 100644 index 23f3758..0000000 --- a/test_data/elaiss1oid_collection/margin_10arcs/hats.properties +++ /dev/null @@ -1,18 +0,0 @@ -#HATS catalog -obs_collection=margin_10arcs -dataproduct_type=margin -hats_nrows=0 -hats_col_ra=RAdeg -hats_col_dec=DEdeg -hats_col_healpix=_healpix_29 -hats_col_healpix_order=29 -hats_primary_table_url=/home/luigi/linea/pzserver_combine_redshift_dedup/test_data/elaiss1oid_collection/catalog -hats_margin_threshold=10.0 -hats_npix_suffix=.parquet -moc_sky_fraction=0.0 -hats_order=2 -hats_builder=hats-import v0.9.0, hats v0.9.0 -hats_creation_date=2026-06-01T19:10UTC -hats_estsize=13 -hats_release_date=2025-08-22 -hats_version=v1.0 diff --git a/test_data/elaiss1oid_collection/margin_10arcs/partition_info.csv b/test_data/elaiss1oid_collection/margin_10arcs/partition_info.csv deleted file mode 100644 index 7117bdc..0000000 --- a/test_data/elaiss1oid_collection/margin_10arcs/partition_info.csv +++ /dev/null @@ -1 +0,0 @@ -Norder,Npix diff --git a/test_data/elaiss1oid_collection/margin_10arcs/properties b/test_data/elaiss1oid_collection/margin_10arcs/properties deleted file mode 100644 index 23f3758..0000000 --- a/test_data/elaiss1oid_collection/margin_10arcs/properties +++ /dev/null @@ -1,18 +0,0 @@ -#HATS catalog -obs_collection=margin_10arcs -dataproduct_type=margin -hats_nrows=0 -hats_col_ra=RAdeg -hats_col_dec=DEdeg -hats_col_healpix=_healpix_29 -hats_col_healpix_order=29 -hats_primary_table_url=/home/luigi/linea/pzserver_combine_redshift_dedup/test_data/elaiss1oid_collection/catalog -hats_margin_threshold=10.0 -hats_npix_suffix=.parquet -moc_sky_fraction=0.0 -hats_order=2 -hats_builder=hats-import v0.9.0, hats v0.9.0 -hats_creation_date=2026-06-01T19:10UTC -hats_estsize=13 -hats_release_date=2025-08-22 -hats_version=v1.0 diff --git a/test_data/jades_dr3_random_sample.parquet b/test_data/jades_dr3_random_sample.parquet deleted file mode 100644 index 5377abb..0000000 Binary files a/test_data/jades_dr3_random_sample.parquet and /dev/null differ diff --git a/test_data/mosdef_final_release_random_sample.parquet b/test_data/mosdef_final_release_random_sample.parquet deleted file mode 100644 index 0268355..0000000 Binary files a/test_data/mosdef_final_release_random_sample.parquet and /dev/null differ diff --git a/test_data/ozdes_dr2_random_sample.parquet b/test_data/ozdes_dr2_random_sample.parquet deleted file mode 100644 index 9a4ca33..0000000 Binary files a/test_data/ozdes_dr2_random_sample.parquet and /dev/null differ diff --git a/test_data/pipeline_generated_sample.parquet b/test_data/pipeline_generated_sample.parquet deleted file mode 100644 index f6d499f..0000000 Binary files a/test_data/pipeline_generated_sample.parquet and /dev/null differ diff --git a/test_data/primus_dr1_random_sample.parquet b/test_data/primus_dr1_random_sample.parquet deleted file mode 100644 index 3bc6c1c..0000000 Binary files a/test_data/primus_dr1_random_sample.parquet and /dev/null differ diff --git a/test_data/test_homogenized_columns_out_of_range.parquet b/test_data/test_homogenized_columns_out_of_range.parquet deleted file mode 100644 index 6767390..0000000 Binary files a/test_data/test_homogenized_columns_out_of_range.parquet and /dev/null differ diff --git a/test_data/test_survey_col_missing.parquet b/test_data/test_survey_col_missing.parquet deleted file mode 100644 index 462dd7d..0000000 Binary files a/test_data/test_survey_col_missing.parquet and /dev/null differ diff --git a/test_data/test_survey_not_supported.parquet b/test_data/test_survey_not_supported.parquet deleted file mode 100644 index 1ff03fe..0000000 Binary files a/test_data/test_survey_not_supported.parquet and /dev/null differ diff --git a/test_data/test_survey_supported_but_wrong_flags.parquet b/test_data/test_survey_supported_but_wrong_flags.parquet deleted file mode 100644 index af1fa09..0000000 Binary files a/test_data/test_survey_supported_but_wrong_flags.parquet and /dev/null differ diff --git a/test_data/vandels_dr4_random_sample.parquet b/test_data/vandels_dr4_random_sample.parquet deleted file mode 100644 index 0e9b0f6..0000000 Binary files a/test_data/vandels_dr4_random_sample.parquet and /dev/null differ diff --git a/test_data/vlt_vimos_v2.0.1_random_sample.parquet b/test_data/vlt_vimos_v2.0.1_random_sample.parquet deleted file mode 100644 index bad39be..0000000 Binary files a/test_data/vlt_vimos_v2.0.1_random_sample.parquet and /dev/null differ diff --git a/test_data/vuds_dr1_random_sample.parquet b/test_data/vuds_dr1_random_sample.parquet deleted file mode 100644 index 53b18a6..0000000 Binary files a/test_data/vuds_dr1_random_sample.parquet and /dev/null differ diff --git a/test_data/vvds_final_release_random_sample.parquet b/test_data/vvds_final_release_random_sample.parquet deleted file mode 100644 index 5c2f2c8..0000000 Binary files a/test_data/vvds_final_release_random_sample.parquet and /dev/null differ diff --git a/test_data/z_cat_CANDELS_clean_sitcomtn-154_random_sample.parquet b/test_data/z_cat_CANDELS_clean_sitcomtn-154_random_sample.parquet deleted file mode 100644 index 5f1fe80..0000000 Binary files a/test_data/z_cat_CANDELS_clean_sitcomtn-154_random_sample.parquet and /dev/null differ diff --git a/test_data/z_cat_NED_clean_sitcomtn-154_random_sample.parquet b/test_data/z_cat_NED_clean_sitcomtn-154_random_sample.parquet deleted file mode 100644 index db6f59b..0000000 Binary files a/test_data/z_cat_NED_clean_sitcomtn-154_random_sample.parquet and /dev/null differ diff --git a/tests/test_all_test_data.py b/tests/test_all_test_data.py deleted file mode 100644 index 71c2460..0000000 --- a/tests/test_all_test_data.py +++ /dev/null @@ -1,106 +0,0 @@ -from __future__ import annotations - -import os -import subprocess -from pathlib import Path - -import pandas as pd -import pytest -import yaml - - -REPO_ROOT = Path(__file__).resolve().parents[1] -CONFIG_PATH = REPO_ROOT / "test_configs" / "config.all.yaml" -RUN_SCRIPT = REPO_ROOT / "run.sh" -ENV_NAME = "pipe_crd" - -EXPECTED_STATS = { - "tie_result": { - 0: 1451, - 1: 16688, - 2: 208, - 3: 351, - }, - "z_flag_homogenized": { - 0.0: 1646, - 1.0: 1793, - 2.0: 1088, - 3.0: 6612, - 4.0: 7208, - 6.0: 351, - }, - "instrument_type_homogenized": { - "g": 1152, - "p": 3838, - "s": 13708, - }, - "compared_to_notna": 3027, -} - - -def _value_counts(df: pd.DataFrame, column: str) -> dict: - return df[column].value_counts(dropna=False).sort_index().to_dict() - - -def _result_stats(df: pd.DataFrame) -> dict: - return { - "tie_result": _value_counts(df, "tie_result"), - "z_flag_homogenized": _value_counts(df, "z_flag_homogenized"), - "instrument_type_homogenized": _value_counts( - df, "instrument_type_homogenized" - ), - "compared_to_notna": int(df["compared_to"].notna().sum()), - } - - -def _env_with_mamba_root_prefix() -> dict[str, str]: - env = os.environ.copy() - env.setdefault("CRC_LOG_LEVEL", "INFO") - - if "MAMBA_ROOT_PREFIX" not in env: - home = Path.home() - for root in (home / "micromamba", home / ".micromamba", home / "miniconda3"): - if (root / "envs" / ENV_NAME).exists(): - env["MAMBA_ROOT_PREFIX"] = str(root) - break - - return env - - -def _temporary_config(tmp_path: Path) -> Path: - with CONFIG_PATH.open() as fp: - config = yaml.safe_load(fp) - - config["output_root_dir"] = str(tmp_path / "storage") - config["output_dir"] = "outputs" - config["param"]["flags_translation_file"] = str(REPO_ROOT / "flags_translation.yaml") - config_path = tmp_path / "config.all.yaml" - with config_path.open("w") as fp: - yaml.safe_dump(config, fp, sort_keys=False) - - return config_path - - -@pytest.mark.slow -def test_all_test_data_pipeline_statistics(tmp_path): - run_dir = tmp_path / "process_all" - config_path = _temporary_config(tmp_path) - - completed = subprocess.run( - [str(RUN_SCRIPT), str(config_path), str(run_dir)], - cwd=REPO_ROOT, - env=_env_with_mamba_root_prefix(), - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - timeout=30 * 60, - check=False, - ) - - assert completed.returncode == 0, completed.stdout[-8000:] - - output_path = run_dir / "crd.parquet" - assert output_path.exists(), f"Missing pipeline output: {output_path}" - - result = pd.read_parquet(output_path) - assert _result_stats(result) == EXPECTED_STATS diff --git a/tests/test_column_mapping.py b/tests/test_column_mapping.py index e91a4ad..24689c2 100644 --- a/tests/test_column_mapping.py +++ b/tests/test_column_mapping.py @@ -15,7 +15,6 @@ from specz import _validate_and_rename # noqa: E402 - LOGGER = logging.getLogger("test.column_mapping") diff --git a/tests/test_crossmatch_adjacency.py b/tests/test_crossmatch_adjacency.py index 1bb6451..ceb661e 100644 --- a/tests/test_crossmatch_adjacency.py +++ b/tests/test_crossmatch_adjacency.py @@ -13,11 +13,17 @@ from crossmatch_auto import ( # noqa: E402 _adjacency_from_pairs as auto_adjacency, +) +from crossmatch_auto import ( _merge_compared_to_partition as auto_merge_compared_to, ) from crossmatch_cross import ( # noqa: E402 _adjacency_from_pairs as cross_adjacency, +) +from crossmatch_cross import ( _attach_distributed_neighbors, +) +from crossmatch_cross import ( _merge_compared_to_partition as cross_merge_compared_to, ) diff --git a/tests/test_crossmatch_diagnostics.py b/tests/test_crossmatch_diagnostics.py index 65cdbbc..1932ce9 100644 --- a/tests/test_crossmatch_diagnostics.py +++ b/tests/test_crossmatch_diagnostics.py @@ -1,8 +1,8 @@ import sys from pathlib import Path -import pandas as pd import dask.dataframe as dd +import pandas as pd sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "packages")) diff --git a/tests/test_crossmatch_saturation.py b/tests/test_crossmatch_saturation.py index 3203a7b..fbf3394 100644 --- a/tests/test_crossmatch_saturation.py +++ b/tests/test_crossmatch_saturation.py @@ -2,8 +2,8 @@ import types from pathlib import Path -import pandas as pd import dask.dataframe as dd +import pandas as pd import pytest sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "packages")) diff --git a/tests/test_dedup_partition_safety.py b/tests/test_dedup_partition_safety.py index e644590..0a642f5 100644 --- a/tests/test_dedup_partition_safety.py +++ b/tests/test_dedup_partition_safety.py @@ -11,11 +11,11 @@ from deduplication import ( # noqa: E402 _collapse_within_dz, - count_global_edge_group_mismatches, - count_global_tie_invariant_violations, _dedup_local_with_margin, _log_representative_radius_diagnostics, _validate_local_tie_invariants, + count_global_edge_group_mismatches, + count_global_tie_invariant_violations, ) @@ -55,6 +55,36 @@ def test_boundary_component_has_same_canonical_group_from_both_pixels(): assert from_b_pixel["tie_result"] == 0 +def test_custom_priority_keeps_star_semantics_without_flag_ranking(): + rows = [ + { + **_row("A", "B, S", 10.0, 4.0), + "custom_score": 1.0, + }, + { + **_row("B", "A, S", 10.0 + 0.1 / 3600.0, 3.0), + "custom_score": 10.0, + }, + { + **_row("S", "A, B", 10.0 + 0.2 / 3600.0, 6.0), + "custom_score": 100.0, + }, + ] + + result = _dedup_local_with_margin( + pd.DataFrame(rows), + pd.DataFrame(), + pixel=None, + tiebreaking_priority=["custom_score"], + instrument_type_priority=None, + group_col="group_id", + ).set_index("CRD_ID") + + assert result["tie_result"].astype(int).to_dict() == {"A": 0, "B": 1, "S": 3} + assert result.loc["A", "group_id"] == result.loc["B", "group_id"] + assert result.loc["S", "group_id"] != result.loc["B", "group_id"] + + def test_local_invariant_accepts_single_winner_and_hard_tie(): frame = pd.DataFrame( { diff --git a/tests/test_deduplication_core.py b/tests/test_deduplication_core.py index 28a01cf..8a82bdb 100644 --- a/tests/test_deduplication_core.py +++ b/tests/test_deduplication_core.py @@ -8,7 +8,6 @@ from deduplication import deduplicate_pandas # noqa: E402 - INSTRUMENT_PRIORITY = {"s": 3, "g": 2, "p": 1} @@ -150,6 +149,24 @@ def test_missing_priority_column_fails_clearly(): ) +def test_missing_semantic_star_flag_fails_clearly(): + frame = pd.DataFrame( + { + "CRD_ID": ["A"], + "compared_to": [pd.NA], + "z": [0.1], + "custom_score": [1.0], + } + ) + + with pytest.raises(KeyError, match="z_flag_homogenized"): + deduplicate_pandas( + frame, + tiebreaking_priority=["custom_score"], + group_col="group_id", + ) + + def test_instrument_priority_mapping_is_required_when_used(): with pytest.raises(ValueError, match="instrument_type_priority is required"): deduplicate_pandas( diff --git a/tests/test_distributed_single_file_output.py b/tests/test_distributed_single_file_output.py new file mode 100644 index 0000000..4505c5a --- /dev/null +++ b/tests/test_distributed_single_file_output.py @@ -0,0 +1,59 @@ +import sys +from pathlib import Path + +import dask.dataframe as dd +import pandas as pd + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "packages")) + +from product_handle import save_dataframe # noqa: E402 + + +def _distributed_frame(): + frame = pd.DataFrame( + { + "CRD_ID": ["A", "B", "C", "D"], + "value": pd.Series([1, 2, 3, 4], dtype="Int64"), + "label": pd.Series(["a", "b", pd.NA, "d"], dtype="string"), + } + ) + return frame, dd.from_pandas(frame, npartitions=3, sort=False) + + +def test_distributed_parquet_export_creates_one_file(tmp_path): + expected, distributed = _distributed_frame() + output_base = tmp_path / "catalog" + + save_dataframe( + distributed, + output_base, + "parquet", + temp_dir=tmp_path / "staging", + ) + + output_path = tmp_path / "catalog.parquet" + assert output_path.is_file() + result = pd.read_parquet(output_path).sort_values("CRD_ID").reset_index(drop=True) + assert result["CRD_ID"].tolist() == expected["CRD_ID"].tolist() + assert result["value"].astype(int).tolist() == [1, 2, 3, 4] + assert not list((tmp_path / "staging").rglob("*.parquet")) + + +def test_distributed_csv_export_creates_one_file_with_one_header(tmp_path): + expected, distributed = _distributed_frame() + output_base = tmp_path / "catalog" + + save_dataframe( + distributed, + output_base, + "csv", + temp_dir=tmp_path / "staging", + ) + + output_path = tmp_path / "catalog.csv" + assert output_path.is_file() + assert output_path.read_text(encoding="utf-8").count("CRD_ID,value,label") == 1 + result = pd.read_csv(output_path).sort_values("CRD_ID").reset_index(drop=True) + assert result["CRD_ID"].tolist() == expected["CRD_ID"].tolist() + assert result["value"].tolist() == [1, 2, 3, 4] + assert not list((tmp_path / "staging").rglob("*.parquet")) diff --git a/tests/test_executor_scaling.py b/tests/test_executor_scaling.py index 31f4218..d67fe3e 100644 --- a/tests/test_executor_scaling.py +++ b/tests/test_executor_scaling.py @@ -29,7 +29,14 @@ def test_slurm_executor_starts_minimum_workers_and_always_enables_adapt(monkeypa ) assert cluster.kwargs["n_workers"] == 11 - assert cluster.adapt_calls == [{"minimum_jobs": 11, "maximum_jobs": 11}] + assert cluster.adapt_calls == [ + { + "minimum_jobs": 11, + "maximum_jobs": 11, + "interval": "10s", + "wait_count": 18, + } + ] def test_slurm_executor_supports_different_adaptive_limits(monkeypatch): @@ -43,9 +50,46 @@ def test_slurm_executor_supports_different_adaptive_limits(monkeypatch): "scale": { "minimum_jobs": 7, "maximum_jobs": 15, + "adaptive_interval_seconds": 5, + "adaptive_scale_down_delay_seconds": 121, + }, + }, + } + ) + + assert cluster.adapt_calls == [ + { + "minimum_jobs": 7, + "maximum_jobs": 15, + "interval": "5s", + "wait_count": 25, + } + ] + + +def test_slurm_executor_supports_legacy_adaptive_wait_count(monkeypatch): + monkeypatch.setattr(executor, "SLURMCluster", FakeSlurmCluster) + + cluster = executor.get_executor( + { + "name": "slurm", + "args": { + "instance": {"cores": 2, "processes": 1, "memory": "20GB"}, + "scale": { + "minimum_jobs": 3, + "maximum_jobs": 22, + "adaptive_interval_seconds": 10, + "adaptive_wait_count": 18, }, }, } ) - assert cluster.adapt_calls == [{"minimum_jobs": 7, "maximum_jobs": 15}] + assert cluster.adapt_calls == [ + { + "minimum_jobs": 3, + "maximum_jobs": 22, + "interval": "10s", + "wait_count": 18, + } + ] diff --git a/tests/test_hats_parquet_staging.py b/tests/test_hats_parquet_staging.py index c6620d3..bcca1f0 100644 --- a/tests/test_hats_parquet_staging.py +++ b/tests/test_hats_parquet_staging.py @@ -31,7 +31,7 @@ def test_staged_parquet_validation_rejects_empty_output( tmp_path / "part-0.parquet", ) - with pytest.raises(RuntimeError, match="empty after HATS parquet staging"): + with pytest.raises(RuntimeError, match="empty after Parquet staging"): _ensure_staged_parquet_non_empty(tmp_path) diff --git a/tests/test_homogenization.py b/tests/test_homogenization.py index e7a13e3..ebe14b1 100644 --- a/tests/test_homogenization.py +++ b/tests/test_homogenization.py @@ -2,6 +2,7 @@ import sys import types from pathlib import Path +from unittest.mock import Mock import dask.dataframe as dd import pandas as pd @@ -13,9 +14,13 @@ tables_io.types = types.SimpleNamespace(PD_DATAFRAME="PD_DATAFRAME") sys.modules["tables_io"] = tables_io +from specz import ( # noqa: E402 + _normalize_custom_tiebreaking_priorities, + _requires_z_flag_homogenization, + validate_combine_configuration, +) from specz_homogenization import _homogenize # noqa: E402 - LOGGER = logging.getLogger("test.homogenization") @@ -96,3 +101,113 @@ def test_user_homogenized_flag_rejects_values_outside_domain(): with pytest.raises(ValueError, match="Invalid values"): _homogenize(frame, config, "demo", LOGGER, type_cast_ok=False) + + +def test_semantic_star_flag_is_preserved_without_being_a_priority(): + frame = dd.from_pandas( + pd.DataFrame( + { + "custom_score": [1.0, 2.0], + "z_flag_homogenized": [4.0, 6.0], + } + ), + npartitions=1, + sort=False, + ) + config = {"tiebreaking_priority": ["custom_score"]} + + result, *_ = _homogenize( + frame, + config, + "demo", + LOGGER, + type_cast_ok=False, + require_z_flag_homogenized=True, + ) + + computed = result.compute() + assert computed["z_flag_homogenized"].tolist() == [4.0, 6.0] + assert "instrument_type_homogenized" not in computed.columns + + +@pytest.mark.parametrize( + ("mode", "cut", "expected"), + [ + ("concatenate", 0, False), + ("concatenate", None, False), + ("concatenate", 3, True), + ("concatenate", "6", True), + ("concatenate", 1.5, False), + ("concatenate", 7, False), + ("concatenate_and_mark_duplicates", 0, True), + ("concatenate_and_remove_duplicates", None, True), + ], +) +def test_flag_homogenization_is_required_by_dedup_or_active_cut( + mode, cut, expected +): + assert _requires_z_flag_homogenization(mode, cut) is expected + + +def test_custom_priority_is_normalized_and_invalid_values_are_logged(): + frame = dd.from_pandas( + pd.DataFrame({"custom_score": ["1.5", "bad", None, 3]}), + npartitions=2, + sort=False, + ) + logger = Mock() + + result = _normalize_custom_tiebreaking_priorities( + frame, ["custom_score"], "demo", logger + ).compute() + + assert str(result["custom_score"].dtype) == "float64" + assert result["custom_score"].tolist()[0] == 1.5 + assert pd.isna(result["custom_score"].tolist()[1]) + assert pd.isna(result["custom_score"].tolist()[2]) + assert result["custom_score"].tolist()[3] == 3.0 + assert logger.info.call_args.args[6] == 1 + + +def test_custom_priority_must_exist_and_have_numeric_values(): + frame = dd.from_pandas( + pd.DataFrame({"other": [1, 2], "invalid_score": ["bad", None]}), + npartitions=1, + sort=False, + ) + + with pytest.raises(ValueError, match="must exist in every input catalog"): + _normalize_custom_tiebreaking_priorities( + frame, ["missing_score"], "demo", LOGGER + ) + with pytest.raises(ValueError, match="no valid numeric values"): + _normalize_custom_tiebreaking_priorities( + frame, ["invalid_score"], "demo", LOGGER + ) + + +def test_combine_configuration_rejects_invalid_mode_and_empty_dedup_priorities(): + with pytest.raises(ValueError, match="Invalid combine_type"): + validate_combine_configuration("unknown", ["score"], 0) + with pytest.raises(ValueError, match="must be non-empty"): + validate_combine_configuration( + "concatenate_and_mark_duplicates", [], 0 + ) + + mode, priorities = validate_combine_configuration("concatenate", [], 0) + assert mode == "concatenate" + assert priorities == [] + + +def test_cut_six_warns_early_for_remove_duplicates(): + logger = Mock() + + validate_combine_configuration( + "concatenate_and_remove_duplicates", + ["z_flag_homogenized"], + 6, + logger, + ) + + logger.warning.assert_called_once() + assert "final catalog will normally be empty" in logger.warning.call_args.args[0] diff --git a/tests/test_previous_results.py b/tests/test_previous_results.py index 0da9054..47926fb 100644 --- a/tests/test_previous_results.py +++ b/tests/test_previous_results.py @@ -7,12 +7,11 @@ import dask.dataframe as dd import pandas as pd - sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "packages")) from specz import ( # noqa: E402 - _drop_previous_results, _copy_extra_columns_from_sources, + _drop_previous_results, _normalize_extra_columns_config, _prefer_pipeline_output_id_mapping, _select_output_columns, diff --git a/tests/test_tie_treatment.py b/tests/test_tie_treatment.py index 48aab26..67b1149 100644 --- a/tests/test_tie_treatment.py +++ b/tests/test_tie_treatment.py @@ -1,12 +1,16 @@ import sys from pathlib import Path +import dask.dataframe as dd import pandas as pd import pytest sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "packages")) -from deduplication import filter_pandas_by_tie_treatment # noqa: E402 +from deduplication import ( # noqa: E402 + filter_dask_by_tie_treatment, + filter_pandas_by_tie_treatment, +) def _results(): @@ -43,8 +47,7 @@ def test_draw_one_keeps_exactly_one_candidate_per_hard_tie(): assert effective == "draw_one" assert resolved == 1 - assert "winner" in set(result["CRD_ID"]) - assert len(set(result["CRD_ID"]) & {"tie-a", "tie-b"}) == 1 + assert set(result["CRD_ID"]) == {"winner", "tie-a"} assert "star" not in set(result["CRD_ID"]) @@ -61,3 +64,29 @@ def test_draw_one_without_group_id_falls_back_to_remove_all(): def test_final_tie_filter_requires_tie_result(): with pytest.raises(RuntimeError, match="tie_result"): filter_pandas_by_tie_treatment(pd.DataFrame({"CRD_ID": ["A"]}), "remove_all") + + +def test_distributed_draw_one_is_lazy_and_deterministic(): + frame = pd.DataFrame( + { + "CRD_ID": ["winner", "loser", "tie-b", "tie-a", "tie-d", "tie-c"], + "group_id": [1, 1, 2, 2, 3, 3], + "tie_result": [1, 0, 2, 2, 2, 2], + } + ) + distributed = dd.from_pandas(frame, npartitions=3, sort=False) + + result, effective = filter_dask_by_tie_treatment(distributed, "draw_one") + + assert isinstance(result, dd.DataFrame) + assert effective == "draw_one" + assert set(result.compute()["CRD_ID"]) == {"winner", "tie-a", "tie-c"} + + +def test_distributed_draw_one_without_group_id_falls_back_to_remove_all(): + distributed = dd.from_pandas(_results().drop(columns="group_id"), npartitions=2) + + result, effective = filter_dask_by_tie_treatment(distributed, "draw_one") + + assert effective == "remove_all" + assert set(result.compute()["CRD_ID"]) == {"winner"}