diff --git a/.github/workflows/publish_dev.yml b/.github/workflows/publish_dev.yml index 59486fb03..3783d6c7c 100644 --- a/.github/workflows/publish_dev.yml +++ b/.github/workflows/publish_dev.yml @@ -21,7 +21,7 @@ jobs: - name: Pull latest changes run: git pull origin dev - name: Install uv - uses: astral-sh/setup-uv@v9 + uses: astral-sh/setup-uv@v10.0.1 with: python-version: '3.10' - name: Configure aws credentials @@ -33,5 +33,5 @@ jobs: - name: Create and upload schemas run: | uv sync --locked - uv run python -m aind_data_schema.utils.json_writer --output $TEMP_DIR --attach-version + uv run python -m biodata_schema.utils.json_writer --output $TEMP_DIR --attach-version uvx --from awscli aws s3 sync $TEMP_DIR s3://${AWS_DATA_SCHEMA_BUCKET}/$S3_PREFIX diff --git a/.github/workflows/run_dev_tests.yml b/.github/workflows/run_dev_tests.yml index 7b84f2ca8..6e5abec41 100644 --- a/.github/workflows/run_dev_tests.yml +++ b/.github/workflows/run_dev_tests.yml @@ -13,11 +13,11 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ '3.11' ] + python-version: [ '3.12' ] steps: - uses: actions/checkout@v3 - name: Install uv - uses: astral-sh/setup-uv@v9 + uses: astral-sh/setup-uv@v10.0.1 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -29,13 +29,14 @@ jobs: strategy: matrix: python-version: [ '3.10', '3.11', '3.12', '3.13' ] + resolution: [ 'lowest-direct', 'highest' ] steps: - uses: actions/checkout@v3 - name: Install uv - uses: astral-sh/setup-uv@v9 + uses: astral-sh/setup-uv@v10.0.1 with: python-version: ${{ matrix.python-version }} - name: Install dependencies - run: uv sync --locked --group dev --group docs + run: uv sync --group dev --group docs --resolution ${{ matrix.resolution }} - name: Run tests and coverage - run: uv run pytest --cov=aind_data_schema --cov=tests --cov-report=term-missing --cov-fail-under=100 + run: uv run pytest --cov=biodata_schema --cov=tests --cov-report=term-missing --cov-fail-under=100 diff --git a/.github/workflows/run_main_tests.yml b/.github/workflows/run_main_tests.yml index 6e118d000..cbcad3ef0 100644 --- a/.github/workflows/run_main_tests.yml +++ b/.github/workflows/run_main_tests.yml @@ -19,7 +19,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: Install uv - uses: astral-sh/setup-uv@v9 + uses: astral-sh/setup-uv@v10.0.1 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -34,13 +34,13 @@ jobs: steps: - uses: actions/checkout@v3 - name: Install uv - uses: astral-sh/setup-uv@v9 + uses: astral-sh/setup-uv@v10.0.1 with: python-version: ${{ matrix.python-version }} - name: Install dependencies run: uv sync --locked --group dev --group docs - name: Run tests and coverage - run: uv run pytest --cov=aind_data_schema --cov=tests --cov-report=term-missing --cov-fail-under=100 + run: uv run pytest --cov=biodata_schema --cov=tests --cov-report=term-missing --cov-fail-under=100 verify_version: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/tag_and_publish_main.yml b/.github/workflows/tag_and_publish_main.yml index 0055f7e8c..a2ff7c8eb 100644 --- a/.github/workflows/tag_and_publish_main.yml +++ b/.github/workflows/tag_and_publish_main.yml @@ -25,7 +25,7 @@ jobs: - name: Push git tag run: git push origin "v${{ env.tag }}" - name: Install uv - uses: astral-sh/setup-uv@v9 + uses: astral-sh/setup-uv@v10.0.1 with: python-version: '3.10' - name: Install dependencies @@ -54,7 +54,7 @@ jobs: - name: Pull latest changes run: git pull origin main - name: Install uv - uses: astral-sh/setup-uv@v9 + uses: astral-sh/setup-uv@v10.0.1 with: python-version: '3.10' - name: Configure aws credentials @@ -66,5 +66,5 @@ jobs: - name: Create and upload schemas run: | uv sync --locked - uv run python -m aind_data_schema.utils.json_writer --output $TEMP_DIR --attach-version + uv run python -m biodata_schema.utils.json_writer --output $TEMP_DIR --attach-version uvx --from awscli aws s3 sync $TEMP_DIR s3://${AWS_DATA_SCHEMA_BUCKET}/$S3_PREFIX diff --git a/.github/workflows/update_dev_docs.yml b/.github/workflows/update_dev_docs.yml index be5cea378..33e99abce 100644 --- a/.github/workflows/update_dev_docs.yml +++ b/.github/workflows/update_dev_docs.yml @@ -14,16 +14,16 @@ jobs: token: ${{ secrets.SERVICE_TOKEN }} ref: ${{ github.head_ref }} - name: Install uv - uses: astral-sh/setup-uv@v9 + uses: astral-sh/setup-uv@v10.0.1 with: python-version: '3.10' - name: Install dependencies run: uv sync --locked --group dev --group docs - name: Generate docs run: | - uv run python src/aind_data_schema/utils/docs/model_generator.py - uv run python src/aind_data_schema/utils/docs/registries_generator.py - uv run python src/aind_data_schema/utils/docs/doc_generator.py + uv run python src/biodata_schema/utils/docs/model_generator.py + uv run python src/biodata_schema/utils/docs/registries_generator.py + uv run python src/biodata_schema/utils/docs/doc_generator.py uv run sphinx-build -b html docs/source/ docs/build/html - name: Commit changes uses: EndBug/add-and-commit@v9 diff --git a/.github/workflows/update_docs.yml b/.github/workflows/update_docs.yml index 7bf8316f7..2a1f82c60 100644 --- a/.github/workflows/update_docs.yml +++ b/.github/workflows/update_docs.yml @@ -15,19 +15,19 @@ jobs: with: token: ${{ secrets.SERVICE_TOKEN }} - name: Install uv - uses: astral-sh/setup-uv@v9 + uses: astral-sh/setup-uv@v10.0.1 with: python-version: '3.10' - name: Install dependencies run: uv sync --locked --group dev - name: Bump schema versions run: | - uv run python -m aind_data_schema.utils.schema_version_bump + uv run python -m biodata_schema.utils.schema_version_bump echo "Finished schema version bump" - name: Update schemas run: | rm -rf $OUTPUT_DIR - uv run python -m aind_data_schema.utils.json_writer --output $OUTPUT_DIR + uv run python -m biodata_schema.utils.json_writer --output $OUTPUT_DIR - name: Create example schemas run: | cd examples @@ -53,7 +53,7 @@ jobs: - name: Pull latest changes run: git pull origin ${{ github.ref_name }} - name: Install uv - uses: astral-sh/setup-uv@v9 + uses: astral-sh/setup-uv@v10.0.1 with: python-version: '3.10' - name: Install dependencies diff --git a/.readthedocs.yaml b/.readthedocs.yaml index f672bdae3..ec58f06b9 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -1,23 +1,23 @@ version: 2 build: - os: ubuntu-22.04 + os: ubuntu-24.04 tools: python: "3.13" nodejs: "20" jobs: - # Build the interactive React Flow schema diagram bundle into - # docs/source/_static/ before Sphinx runs (the bundle is git-ignored and - # regenerated here; the JSON it fetches is generated by conf.py). + # Build the interactive React Flow schema diagram before Sphinx runs. pre_build: - npm --prefix diagram-app ci - npm --prefix diagram-app run build - pre_install: - - python -m pip install uv - - uv sync --locked --group dev --group docs python: - install: [] + install: + - method: uv + command: sync + groups: + - dev + - docs sphinx: # Path to your Sphinx configuration file. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6a1e3ee8c..fff4608b9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -39,9 +39,9 @@ Documentation is automatically built when you open a PR into the biodata-schema To generate the source files for the documentation and model class links, run: ```bash -uv run python src/aind_data_schema/utils/docs/model_generator.py -uv run python src/aind_data_schema/utils/docs/registries_generator.py -uv run python src/aind_data_schema/utils/docs/doc_generator.py +uv run python src/biodata_schema/utils/docs/model_generator.py +uv run python src/biodata_schema/utils/docs/registries_generator.py +uv run python src/biodata_schema/utils/docs/doc_generator.py ``` The front page embeds an interactive React Flow diagram of the `Metadata` schema (`diagram-app/`). Build its JS/CSS bundle once before building the docs (it's git-ignored and only needs rebuilding when `diagram-app/` changes): @@ -73,7 +73,7 @@ Testing is required to open a PR in this repository to ensure robustness and rel - Please test your changes using pytest and coverage: ```bash - uv run pytest --cov=aind_data_schema --cov=tests --cov-report=term-missing + uv run pytest --cov=biodata_schema --cov=tests --cov-report=term-missing ``` To open the coverage report in a browser, you can run @@ -110,7 +110,7 @@ Use `uv run ruff format .` to apply Ruff's formatting. **NOTE**: Please note that these linters are automatically run in github actions when a PR is opened. These linters must pass for a PR to merge. ### Units -Unit types (i.e. anything from [aind_data_schema_models.units](https://github.com/AllenNeuralDynamics/aind-data-schema-models/blob/main/src/aind_data_schema_models/units.py)) should always be paired with a variable in one of two patterns. +Unit types (i.e. anything from [biodata_models.units](https://github.com/AllenNeuralDynamics/biodata-models/blob/main/src/biodata_models/units.py)) should always be paired with a variable in one of two patterns. When you have a single `variable` with a unit, you should add the `_unit` suffix on the name of the unit: @@ -157,7 +157,7 @@ When you are ready to open a pull request, please link any relevant issues and r ## Release - From dev, create a branch called release-vX.Y.Z -- Manually increment the version number in the aind_data_schema/__init__.py file to match +- Manually increment the version number in the biodata_schema/__init__.py file to match - Manually increment the major/minor/patch versions of the core files as needed - Push the branch and open a PR into main - After this push, any last minute changes to the release-vX.Y.Z will have to done to via a PR diff --git a/V3-MIGRATION-GUIDE.md b/V3-MIGRATION-GUIDE.md new file mode 100644 index 000000000..25b5d61bc --- /dev/null +++ b/V3-MIGRATION-GUIDE.md @@ -0,0 +1,454 @@ +# biodata-schema v3.0.0 migration guide + +Version 3 removes fields, classes, and methods that version 2 marked as deprecated. Version +2 accepted these values and issued warnings. Version 3 rejects them with Pydantic's +`ValidationError` (`extra_forbidden`) because schema models reject extra fields. + +The `aind-metadata-upgrader` package handles automatic metadata upgrades separately. + +## Contents + +1. [`coordinate_system` renamed to `global_`/`local_coordinate_system`](#1-coordinate_system-renamed) +2. [`Section` coordinate fields moved to `PlanarSection`](#2-section-coordinate-fields) +3. [Device fields replaced by `shape`](#3-device-size-fields-replaced-by-shape) +4. [`DAQChannel.channel_index`](#4-daqchannelchannel_index) +5. [`BreedingInfo.breeding_group`](#5-breedinginfobreeding_group) +6. [Olfactory stimulus classes](#6-olfactory-stimulus-classes) +7. [`DataDescription.from_*` classmethods](#7-datadescriptionfrom_-classmethods) +8. [`QCMetric.tags` list form](#8-qcmetrictags-list-form) +9. [`CoordinateSystemLibrary` removed](#9-coordinatesystemlibrary-removed) +10. [Removed helper](#10-removed-helper) +11. [Warnings now raise errors](#11-warnings-promoted-to-errors) +12. [Warnings that remain](#12-validators-still-emitting-warnings) +13. [Stop using `DEPTH` axes](#13-stop-using-depth-axes) + +--- + +## 1. `coordinate_system` renamed + +Version 2 split `coordinate_system` into `global_coordinate_system` for top-level and +container models, and `local_coordinate_system` for device and config models. Version 3 +removes the old field and the validators that copied its value forward. + +Please see the [coordinate systems page on the documentation](https://biodata-schema.readthedocs.io/en/latest/coordinate_systems.html) for more details about coordinate systems specifically. + +### Use `global_coordinate_system` for these models + +| Model | Module | +| --- | --- | +| `Acquisition` | `core/acquisition.py` | +| `Procedures` | `core/procedures.py` | +| `Instrument` | `core/instrument.py` | +| `Surgery` | `components/subject_procedures.py` | +| `PlanarSectioning` | `components/specimen_procedures.py` | + +### Use `local_coordinate_system` for these models + +| Model | Module | +| --- | --- | +| `DevicePosition` and subclasses such as `Monitor` | `components/devices.py` | +| `ImagingConfig` | `components/configs.py` | +| `LickSpoutConfig` | `components/configs.py` | +| `AirPuffConfig` | `components/configs.py` | +| `ManipulatorConfig` | `components/configs.py` | +| `ProbeConfig` | `components/configs.py` | + +**Before** + +```python +Acquisition(..., coordinate_system=CoordinateSystemLibrary.BREGMA_ARI) +ProbeConfig(..., coordinate_system=CoordinateSystemLibrary.BREGMA_ARI) +``` + +**After** + +Here `BREGMA_ARI` is a coordinate system defined by your project, as shown in +[section 9](#9-coordinatesystemlibrary-removed). + +```python +Acquisition(..., global_coordinate_system=BREGMA_ARI) +ProbeConfig(..., local_coordinate_system=BREGMA_ARI) +``` + +Keep `AtlasCoordinate.coordinate_system`. It stores an `Atlas` and does not use the renamed +fields. + +Set `Instrument.global_coordinate_system`, `ManipulatorConfig.local_coordinate_system`, and +`ProbeConfig.local_coordinate_system` explicitly. + +## 2. `Section` coordinate fields + +Use `PlanarSection` for sections with coordinate data. Version 3 removes these fields from +`Section` (`components/specimen_procedures.py`): + +- `coordinate_system_name` +- `start_coordinate` +- `end_coordinate` +- `thickness` +- `thickness_unit` +- `partial_slice` + +Version 3 also removes the `deprecated_coordinate_fields` validator. + +**Before** + +```python +Section(output_specimen_id="123456_001", coordinate_system_name="BREGMA_ARI", thickness=0.1) +``` + +**After** + +```python +PlanarSection( + output_specimen_id="123456_001", + coordinate_system_name="BREGMA_ARI", + start_coordinate=Translation(translation=[0.3, 0, 0]), + thickness=0.1, + thickness_unit=SizeUnit.MM, +) +``` + +`PlanarSection` requires `coordinate_system_name`, `start_coordinate`, and either +`end_coordinate` or `thickness`. When supplying `thickness`, also supply `thickness_unit`. + +## 3. Device size fields replaced by `shape` + +Use a geometry object instead of the `Scale`-based `size`/`size_unit` pair on `Enclosure` and +`Arena` (`components/devices.py`). + +| Model | Remove | Use | +| --- | --- | --- | +| `Enclosure` | `size`, `size_unit` | `shape: Rectangle \| Circle` | +| `Arena` | `size`, `size_unit` | `shape: Circle \| Rectangle` | + +Both models require `shape`. + +**Before** + +```python +Arena(..., size=Scale(scale=[30, 30, 20]), size_unit=SizeUnit.CM) +``` + +**After** + +```python +Arena(..., shape=Rectangle(width=30, height=30, size_unit=SizeUnit.CM)) +``` + +Keep the other `size_unit` fields in the schema, including those on `Monitor`, `Rectangle`, +and `Wheel`. + +## 4. `DAQChannel.channel_index` + +Remove `DAQChannel.channel_index` and its `deprecated_channel_index` validator from +`components/devices.py`. Use `DAQChannel.port` instead. + +**Before** + +```python +DAQChannel(channel_name="ch", channel_type=DaqChannelType.DI, channel_index=1) +``` + +**After** + +```python +DAQChannel(channel_name="ch", channel_type=DaqChannelType.DI, port=1) +``` + +Keep `OlfactometerChannel.channel_index` and `OlfactometerChannelInfo.channel_index`. +Those fields serve different purposes. + +## 5. `BreedingInfo.breeding_group` + +Remove `BreedingInfo.breeding_group` and the `warn_breeding_group_deprecated` validator from +`components/subjects.py`. + +## 6. Olfactory stimulus classes + +Version 3 removes these classes from `components/stimulus.py`: + +| Remove | Use | +| --- | --- | +| `OlfactometerChannelConfig` | `OlfactometerChannelInfo` in `components/configs.py` | +| `OlfactoryStimulation` | `StimulusEpoch.stimulus_name` and `OlfactometerConfig` in `components/configs.py` | + +Remove uses of `OlfactoryStimulation.channels` and `OlfactoryStimulation.notes` too. + +## 7. `DataDescription.from_*` classmethods + +Replace the three `DataDescription` derivation classmethods from `core/data_description.py` +with the functions in `biodata_schema.utils.inheritance`: + +| Remove | Use | +| --- | --- | +| `DataDescription.from_raw(...)` | `derive_data_description_from_raw(...)` | +| `DataDescription.from_derived(...)` | `derive_data_description_from_derived(...)` | +| `DataDescription.from_data_description(...)` | `derive_data_description(...)` | + +The signatures stay the same. Change the import and function name: + +```python +from biodata_schema.utils.inheritance import ( + derive_data_description, + derive_data_description_from_derived, + derive_data_description_from_raw, +) + +derived = derive_data_description_from_raw(dd, "spikesort-ks25", creation_time=dt) +``` + +Or use the `Metadata.from_metadata` helpers. + +## 8. `QCMetric.tags` list form + +Use a dictionary for `QCMetric.tags`. Version 3 removes the `fix_tag_lists` validator from +`core/quality_control.py`, so list-valued `tags` now fail validation. + +**Before (v2.2.x JSON)** + +```json +{"tags": ["Probe A", "Drift"]} +``` + +**After** + +```json +{"tags": {"probe": "Probe A", "issue": "Drift"}} +``` + +### Related removal: `fix_default_grouping_list` + +Version 3 removes `QualityControl.fix_default_grouping_list`. Pass `default_grouping` as a +list of strings or for multi-branch hierarchies a list of tuples of strings. + +## 9. `CoordinateSystemLibrary` removed + +Version 3 removes `biodata_schema.components.coordinates.CoordinateSystemLibrary` and its +fixed set of named `CoordinateSystem` constants, including `BREGMA_ARI`, `BREGMA_RAS`, +`BREGMA_ARID`, `BREGMA_RASD`, `ARENA_RBT`, `SIPE_CAMERA_RBF`, `SIPE_MONITOR_RTF`, +`SIPE_SPEAKER_LTF`, `MPM_MANIP_RFB`, `PINPOINT_PROBE_RSAB`, `SPIM_RPI`, `SPIM_IJK`, +`MRI_LPS`, and `IMAGE_XYZ`. + +Define the coordinate systems for your rig or project in your own module. Import them where +you need them. `CoordinateSystem`, `Axis`, `Origin`, `AxisName`, and `Direction` still work. + +**Before** + +```python +from biodata_schema.components.coordinates import CoordinateSystemLibrary + +Acquisition(..., global_coordinate_system=CoordinateSystemLibrary.BREGMA_ARI) +``` + +**After** + +```python +from biodata_models.coordinates import AxisName, Direction, Origin +from biodata_models.units import SizeUnit + +from biodata_schema.components.coordinates import Axis, CoordinateSystem + +BREGMA_ARI = CoordinateSystem( + name="BREGMA_ARI", + origin=Origin.BREGMA, + axis_unit=SizeUnit.MM, + axes=[ + Axis(name=AxisName.AP, direction=Direction.PA), + Axis(name=AxisName.ML, direction=Direction.LR), + Axis(name=AxisName.SI, direction=Direction.SI), + ], +) + +Acquisition(..., global_coordinate_system=BREGMA_ARI) +``` + +For an old definition, see `components/coordinates.py` at the v2.9.0 tag. The examples in +`examples/` also define coordinate systems inline. + +Keep using `AtlasLibrary`; version 3 still ships it. + +## 10. Removed helper + +Version 3 removes `biodata_schema.base.migrate_deprecated_coordinate_system`. No +replacement exists because the helper only copied the old coordinate field forward. + +`recursive_get_all_names` in `utils/validators.py` now handles the fields directly and no +longer skips `coordinate_system`. + +### The `transforms` extra is gone + +Do not use `pip install biodata-schema[transforms]`. Version 3 removes that extra and its +`scipy` dependency. + +## 11. Warnings now raise errors + +Version 3 turns these five v2 warnings into validation errors. + +### `Code` requires `commit_hash` or `version` + +Set at least one of these fields. `Code._ensure_commit_hash_or_version` in +`components/identifiers.py` now enforces the requirement. + +```python +Code(url="https://github.com/AllenNeuralDynamics/example") # ValidationError +Code(url="https://github.com/AllenNeuralDynamics/example", version="0.0.1") # ok +``` + +### Wrap `Injection` procedures + +`Procedures.reject_injections` in `core/procedures.py` now rejects bare `Injection` objects. +Wrap each injection in a `Surgery` or `NonSurgicalInjection`: + +```python +Procedures(subject_id="12345", subject_procedures=[Injection(...)]) # ValidationError +Procedures(subject_id="12345", subject_procedures=[Surgery(procedures=[Injection(...)])]) # ok +``` + +### Give `Instrument` components unique names + +`Instrument.validate_unique_component_names` in `core/instrument.py` now raises an error. +Give unique names to: + +- entries in `Instrument.components` +- different objects anywhere in the component tree + +You can reference one shared object from multiple places. The validator compares object +values before it reports a duplicate name. + +The validator only checks names on `Device` and `Assembly` objects and their subclasses. +Names on coordinate systems, software, and other non-device/non-assembly objects are +ignored. You can reuse a package across devices, and a software name can match a device +name. For example, this remains valid: + +```python +Software(name="Bonsai", version="2.5") +``` + +Keep device names unique because `Connection.source_device` and `Connection.target_device` +find components by name. + +#### Renamed devices in `multiplane_ophys_instrument` + +Keep each `CameraAssembly` name and give its inner camera a distinct name: + +| Assembly name | Inner camera name | +| --- | --- | +| `Behavior Camera` | `Behavior Camera Detector` | +| `Eye Camera` | `Eye Camera Detector` | +| `Face Camera` | `Face Camera Detector` | + +Apply the same rule when an assembly and its inner camera share a name: keep the assembly +name because connections use it, and rename the inner device. + +### Specify monitor units + +Set `contrast_unit` or `brightness_unit` whenever you set the matching value. Version 3 +removed `Monitor.add_units_if_needed`, which filled in `percent` automatically. + +```python +Monitor(..., contrast=50) # ValidationError +Monitor(..., contrast=50, contrast_unit=UnitlessUnit.PERCENT) # ok +``` + +### Use `calibration` as the `CalibrationObject` subject ID + +`CalibrationObject` subjects must use `"calibration"` as their `subject_id`. +`Metadata.validate_calibration_object_tags` raises a validation error for any other value: + +```python +Subject( + subject_id="calibration", + subject_details=CalibrationObject(...), +) +``` + +## 12. Warnings that remain + +Version 3 removes all `DeprecationWarning`s. The following validators and helpers still emit +warnings. + +### `UserWarning`, visible through `warnings` and `pytest.warns` + +| Location | Validator | Fires when | Reason | +| --- | --- | --- | --- | +| `core/metadata.py` | `Metadata.validate_expected_files_by_modality` | A required file is missing | Metadata can arrive in pieces. | +| `core/metadata.py` | `Metadata.validate_training_protocol_references` | `StimulusEpoch.training_protocol_name` has no match in `Procedures` | The reference matters only when both files exist. | +| `core/metadata.py` | `Metadata.validate_data_description_name_time_consistency` | The name's creation time differs greatly from `acquisition_end_time` | The check uses a heuristic. | +| `core/processing.py` | `Processing.__add__` | Merged objects share process names | The merge renames the processes. | +| `base.py` | `_GenericModel.validate_fieldnames` | A field name contains `.` or `$` | The check covers user-supplied `GenericModel` data. | + +### `logger.warning`, log output only + +| Location | Validator | Fires when | Reason | +| --- | --- | --- | --- | +| `core/metadata.py` | `Metadata.validate_core_fields` | A core field fails its own validation | The helper can wrap partially invalid cores. | +| `core/acquisition.py` | `DataStream.__add__` | Two overlapping streams contain the same active device | The duplicate device is removed; only DAQ devices should be shared between overlapping streams. | +| `utils/validators.py` | `recursive_check_paths` | An `AssetPath` does not exist on disk | The working directory controls path resolution. | +| `base.py` | `DataCoreModel.write_standard_file` | A serialized file exceeds `MAX_FILE_SIZE` (500 KB) | The write path warns instead of discarding data. | + +## 13. Stop using `DEPTH` axes + +Do not add an `AxisName.DEPTH` axis to a `CoordinateSystem`. + +Record insertion depth as a **local translation along the device's own +depth axis** instead. + +The coordinate system builder's probe example rotates the probe locally, translates +to the entry point in the global frame, then inserts along the probe's local Y axis. +The local positive Y axis points up the probe, so insertion uses a negative translation. + +```python +from biodata_models.coordinates import AxisName, Direction, Origin +from biodata_models.units import AngleUnit, SizeUnit +from biodata_schema.components.coordinates import ( + Axis, + CoordinateSystem, + Handedness, + ReferenceCoordinateSystem, + Rotation, + Translation, +) + +instrument_cs = CoordinateSystem( + name="BREGMA_RAS", + origin=Origin.BREGMA, + axis_unit=SizeUnit.MM, + handedness=Handedness.RIGHT, + axes=[ + Axis(name=AxisName.ML, direction=Direction.LR), + Axis(name=AxisName.AP, direction=Direction.PA), + Axis(name=AxisName.SI, direction=Direction.IS), + ], +) + +device_cs = CoordinateSystem( + name="Probe", + origin=Origin.TIP, + axis_unit=SizeUnit.MM, + handedness=Handedness.LEFT, + axes=[ + Axis(name=AxisName.X, direction=Direction.AP), + Axis(name=AxisName.Y, direction=Direction.DU), + Axis(name=AxisName.Z, direction=Direction.LR), + ], +) + +transform = [ + Rotation( + angles=[-15, 0, 0], + angles_unit=AngleUnit.DEG, + reference_coordinate_system=ReferenceCoordinateSystem.LOCAL, + ), + Translation(translation=[1.2, -2, -0.5]), + Translation( + translation=[0, -4, 0], + reference_coordinate_system=ReferenceCoordinateSystem.LOCAL, + ), +] +``` + +Use `instrument_cs` as the acquisition's `global_coordinate_system`, `device_cs` as +the `ProbeConfig.local_coordinate_system`, and `transform` as `ProbeConfig.transform`. + +See the [coordinate systems page](https://biodata-schema.readthedocs.io/en/latest/coordinate_systems.html) +and the examples in `examples/` for the full pattern. diff --git a/diagram-app/src/types.ts b/diagram-app/src/types.ts index 3aa2f32ae..9441f5cb8 100644 --- a/diagram-app/src/types.ts +++ b/diagram-app/src/types.ts @@ -1,4 +1,4 @@ -/** Mirrors the JSON produced by aind_data_schema.utils.schema_tree.generate_schema_diagram_data */ +/** Mirrors the JSON produced by biodata_schema.utils.schema_tree.generate_schema_diagram_data */ export interface FieldLink { /** Model key to drill into in-app, or null if this type isn't a local model (e.g. an enum). */ diff --git a/diagram-app/vite.config.ts b/diagram-app/vite.config.ts index 4fc65d68f..009559284 100644 --- a/diagram-app/vite.config.ts +++ b/diagram-app/vite.config.ts @@ -5,7 +5,7 @@ import { dirname, resolve } from "node:path"; import { existsSync, readFileSync } from "node:fs"; const root = dirname(fileURLToPath(import.meta.url)); -// The JSON is generated by `aind_data_schema.utils.schema_tree` at Sphinx +// The JSON is generated by `biodata_schema.utils.schema_tree` at Sphinx // build time, into the same directory this app's bundle is built into. const SCHEMA_JSON = resolve(root, "../docs/source/_static/schema-diagram/schema_diagram.json"); @@ -20,7 +20,7 @@ function schemaJsonDevPlugin(): Plugin { if (!existsSync(SCHEMA_JSON)) { res.statusCode = 404; return res.end( - "schema_diagram.json not found — run `python -m aind_data_schema.utils.schema_tree` " + + "schema_diagram.json not found — run `python -m biodata_schema.utils.schema_tree` " + "or the diagram data generator to produce it first.", ); } diff --git a/docs/base/core/acquisition.md b/docs/base/core/acquisition.md index a1298bcf2..8437e130c 100644 --- a/docs/base/core/acquisition.md +++ b/docs/base/core/acquisition.md @@ -1,6 +1,6 @@ # Acquisition -[Link to code](https://github.com/AllenNeuralDynamics/biodata-schema/blob/dev/src/aind_data_schema/core/acquisition.py) +[Link to code](https://github.com/AllenNeuralDynamics/biodata-schema/blob/dev/src/biodata_schema/core/acquisition.py) An acquisition is single episode of data collection that creates one data asset. diff --git a/docs/base/core/data_description.md b/docs/base/core/data_description.md index 4014ae1c9..602e7b3ad 100644 --- a/docs/base/core/data_description.md +++ b/docs/base/core/data_description.md @@ -1,6 +1,6 @@ # Data description -[Link to code](https://github.com/AllenNeuralDynamics/biodata-schema/blob/dev/src/aind_data_schema/core/data_description.py) +[Link to code](https://github.com/AllenNeuralDynamics/biodata-schema/blob/dev/src/biodata_schema/core/data_description.py) The `data_description.json` file tracks administrative information about a data asset, including affiliated researchers/organizations, projects, data modalities, dates of collection, and more. diff --git a/docs/base/core/instrument.md b/docs/base/core/instrument.md index fa741e6e1..217f9bc8f 100644 --- a/docs/base/core/instrument.md +++ b/docs/base/core/instrument.md @@ -1,6 +1,6 @@ # Instrument -[Link to code](https://github.com/AllenNeuralDynamics/biodata-schema/blob/dev/src/aind_data_schema/core/instrument.py) +[Link to code](https://github.com/AllenNeuralDynamics/biodata-schema/blob/dev/src/biodata_schema/core/instrument.py) The `instrument.json` collects the components, mostly hardware devices, used to collect data. In general, the instrument schema describes the static state of the data acquisition hardware across sessions. The [Acquisition](acquisition.md) is used to describe the configuration of components for a specific session. @@ -26,22 +26,22 @@ If you need to specify additional information about a device we will need to add ### Missing organizations -You can find the full list of [Organizations](aind_data_schema_models/organizations.md) in the `aind-data-schema-models` repository. Some device types are restricted to a subset of this full list to simplify the `metadata-entry` app. Please open an [issue](https://github.com/AllenNeuralDynamics/biodata-schema/issues) if you need a manufacturer that isn't available in either the main list or one of the subsets. +You can find the full list of [Organizations](biodata_models/organizations.md) in the `biodata-models` repository. Some device types are restricted to a subset of this full list to simplify the `metadata-entry` app. Please open an [issue](https://github.com/AllenNeuralDynamics/biodata-schema/issues) if you need a manufacturer that isn't available in either the main list or one of the subsets. ## Position -### RelativePosition +### Relative position -For all devices where a position is expected you are *required* to provide the relative position. This is a `List[AnatomicalRelative]`, for example you can specify that a computer monitor is `[Anterior]`. Relative positions should be used for devices that might have small position adjustments made from day-to-day and where the exact position is not important. +For all devices where a position is expected you are *required* to provide the `relative_position` field. This is a `List[AnatomicalRelative]`, for example you can specify that a computer monitor is `[Anterior]`. Relative positions should be used for devices that might have small position adjustments made from day-to-day and where the exact position is not important. ### Exact position -For devices where you know the exact position you need to describe the `CoordinateSystem` and and `transform` of the device. The transform describes the `device_to_instrument` transformation, i.e. given a point in the device's coordinate system (0,0,0) how do you need to translate (and rotate/scale) that point to place it in the instrument's coordinate system. Please refer to the [coordinate systems](coordinate_systems.md) page for additional details. +For devices where you know the exact position you need to describe the `local_coordinate_system` and `transform` of the device. The transform describes the `device_to_instrument` transformation, i.e. given a point in the device's coordinate system (0,0,0) how do you need to translate (and rotate/scale) that point to place it in the instrument's coordinate system. Please refer to the [coordinate systems](coordinate_systems.md) page for additional details. ## Examples - [Ephys instrument](https://github.com/AllenNeuralDynamics/biodata-schema/blob/dev/examples/ephys_instrument.py) -- [AIBS Smartspim instrument](https://github.com/AllenNeuralDynamics/biodata-schema/blob/dev/examples/aibs_smartspim.py) +- [AIBS Smartspim instrument](https://github.com/AllenNeuralDynamics/biodata-schema/blob/dev/examples/aibs_smartspim_instrument.py) - [AIND Smartspim instrument](https://github.com/AllenNeuralDynamics/biodata-schema/blob/dev/examples/aind_smartspim_instrument.py) - [Exaspim instrument](https://github.com/AllenNeuralDynamics/biodata-schema/blob/dev/examples/exaspim_instrument.py) - [FIP / Ophys instrument](https://github.com/AllenNeuralDynamics/biodata-schema/blob/dev/examples/fip_ophys_instrument.py) diff --git a/docs/base/core/metadata.md b/docs/base/core/metadata.md index f40124fec..564284691 100644 --- a/docs/base/core/metadata.md +++ b/docs/base/core/metadata.md @@ -1,5 +1,5 @@ # Metadata -[Link to code](https://github.com/AllenNeuralDynamics/biodata-schema/blob/dev/src/aind_data_schema/core/metadata.py) +[Link to code](https://github.com/AllenNeuralDynamics/biodata-schema/blob/dev/src/biodata_schema/core/metadata.py) The `metadata.json` gathers together the other metadata files and adds information about where the data asset is stored. diff --git a/docs/base/core/model.md b/docs/base/core/model.md index d24a27bd3..1a85668f0 100644 --- a/docs/base/core/model.md +++ b/docs/base/core/model.md @@ -1,6 +1,6 @@ # Model -[Link to code](https://github.com/AllenNeuralDynamics/biodata-schema/blob/dev/src/aind_data_schema/core/model.py) +[Link to code](https://github.com/AllenNeuralDynamics/biodata-schema/blob/dev/src/biodata_schema/core/model.py) The Model metadata schema is an extension of the Processing schema tailored to model weights and other data and code artifacts underlying machine learning models - these may be trained on one dataset and evaluated on others, and may be intended to undergo further training iteratively in future versions. diff --git a/docs/base/core/procedures.md b/docs/base/core/procedures.md index 25bd479ed..8e4fac08c 100644 --- a/docs/base/core/procedures.md +++ b/docs/base/core/procedures.md @@ -1,6 +1,6 @@ # Procedures -[Link to code](https://github.com/AllenNeuralDynamics/biodata-schema/blob/dev/src/aind_data_schema/core/procedures.py) +[Link to code](https://github.com/AllenNeuralDynamics/biodata-schema/blob/dev/src/biodata_schema/core/procedures.py) The `procedures.json` file contains anything done to the subject or specimen prior to data collection. This can include surgeries, injections, tissue processing, sectioning, immunolabeling, etc. The procedures metadata also contains implanted devices and their configurations, for example for chronic insertions. diff --git a/docs/base/core/processing.md b/docs/base/core/processing.md index 14471b43d..70f73318b 100644 --- a/docs/base/core/processing.md +++ b/docs/base/core/processing.md @@ -1,6 +1,6 @@ # Processing -[Link to code](https://github.com/AllenNeuralDynamics/biodata-schema/blob/dev/src/aind_data_schema/core/processing.py) +[Link to code](https://github.com/AllenNeuralDynamics/biodata-schema/blob/dev/src/biodata_schema/core/processing.py) The `processing.json` file captures the data processing and analysis steps that have been carried out – mostly for derived data assets. This tracks what code was used for each step, when it was run, what the input and outputs where, what parameters were set. This includes things like spike sorting, image alignment, cell segmentation. It also includes manual annotation, quality control, and data analysis. diff --git a/docs/base/core/quality_control.md b/docs/base/core/quality_control.md index c5e47d40f..a9e99065d 100644 --- a/docs/base/core/quality_control.md +++ b/docs/base/core/quality_control.md @@ -1,12 +1,12 @@ # Quality control -[Link to code](https://github.com/AllenNeuralDynamics/biodata-schema/blob/dev/src/aind_data_schema/core/quality_control.py) +[Link to code](https://github.com/AllenNeuralDynamics/biodata-schema/blob/dev/src/biodata_schema/core/quality_control.py) Quality control is a collection of **metrics** evaluated on a data asset. [QCMetric](#qcmetric) objects should be generated during pipelines: from raw data, during processing, and during analysis by researchers. -Every [QCMetric](#qcmetric) has a `aind_data_schema.quality_control.State` which takes the value of the metric and compares it to some rule. Metrics can only pass or fail. Metrics that require manual evaluation are set to pending. +Every [QCMetric](#qcmetric) has a `Status` which takes the value of the metric and compares it to some rule. Metrics can only pass or fail. Metrics that require manual evaluation are set to pending. ## Details @@ -16,7 +16,7 @@ Each [QCMetric](#qcmetric) is a single value or array of values that can be comp Each [QCMetric](#qcmetric) has a [Status](#status). The [Status](#status) should depend directly on the `QCMetric.value`, either by a simple function: "value>5", or by a qualitative rule: "Field of view includes visual areas". The `QCMetric.description` field should describe the rule used to set the status. Metrics can be evaluated multiple times, in which case the new status should be appended the `QCMetric.status_history`. -Each [QCMetric](#qcmetric) is annotated with three pieces of additional metadata: the [Stage](#stage) during which it was evaluated, the [Modality](aind_data_schema_models/modalities.md#modality) of the evaluated data, and [tags](#tags). +Each [QCMetric](#qcmetric) is annotated with three pieces of additional metadata: the [Stage](#stage) during which it was evaluated, the [Modality](biodata_models/modalities.md#modality) of the evaluated data, and [tags](#tags). ### Curations diff --git a/docs/base/core/subject.md b/docs/base/core/subject.md index 941b165e3..f8f08a033 100644 --- a/docs/base/core/subject.md +++ b/docs/base/core/subject.md @@ -1,6 +1,6 @@ # Subject -[Link to code](https://github.com/AllenNeuralDynamics/biodata-schema/blob/dev/src/aind_data_schema/core/subject.py) +[Link to code](https://github.com/AllenNeuralDynamics/biodata-schema/blob/dev/src/biodata_schema/core/subject.py) The `subject.json` file describes the subject from which data was obtained. diff --git a/docs/source/_static/coordinate_system_builder.html b/docs/source/_static/coordinate_system_builder.html index ab9b65196..dbdd67d9f 100644 --- a/docs/source/_static/coordinate_system_builder.html +++ b/docs/source/_static/coordinate_system_builder.html @@ -17,7 +17,7 @@ body { display: flex; min-width: 700px; min-height: 600px; height: 100vh; font: 13px/1.5 system-ui, sans-serif; background: #f5f5f5; overflow: auto; } #sidebar { - width: 360px; min-width: 300px; flex-shrink: 0; overflow-y: auto; overflow-x: hidden; + width: 400px; min-width: 340px; flex-shrink: 0; overflow-y: auto; overflow-x: hidden; background: #fff; border-right: 1px solid #ddd; padding: 12px; display: flex; flex-direction: column; gap: 12px; } @@ -25,6 +25,7 @@ h2 { font-size: 11px; font-weight: 700; text-transform: uppercase; letter-spacing: .05em; color: #888; margin-bottom: 6px; } .section { border: 1px solid #e0e0e0; border-radius: 6px; flex-shrink: 0; } + .section[hidden] { display: none; } .section-header { padding: 8px 10px; background: #fafafa; border-bottom: 1px solid #e0e0e0; font-weight: 600; font-size: 12px; color: #444; @@ -100,6 +101,32 @@ padding: 8px 10px; border-radius: 4px; line-height: 2.0; border: 1px solid #ddd; max-width: 240px; } + #mode-nav { + position: absolute; right: 18px; bottom: 18px; + display: flex; align-items: center; gap: 8px; + padding: 7px 8px; background: rgba(255,255,255,.94); + border: 1px solid #d8d8d8; border-radius: 6px; + box-shadow: 0 2px 8px rgba(0,0,0,.08); + } + #mode-nav[hidden] { display: none; } + .mode-nav-btn { + width: 30px; height: 30px; padding: 0; + border: 1px solid #cfcfcf; border-radius: 4px; + background: #fff; color: #333; cursor: pointer; + font-size: 18px; line-height: 1; + } + .mode-nav-btn:hover:not(:disabled) { background: #f0f0f0; } + .mode-nav-btn:disabled { color: #bbb; cursor: default; } + #mode-nav-copy { min-width: 150px; text-align: center; } + #mode-nav-title { display: block; font-size: 11px; font-weight: 700; color: #444; } + #mode-nav-detail { display: block; font-size: 10px; color: #888; } + body.global-single-mode #sidebar { width: 430px; min-width: 380px; padding: 18px; gap: 16px; } + body.global-single-mode #global-section .section-header { padding: 11px 14px; font-size: 13px; } + body.global-single-mode #global-section .section-body { padding: 14px; gap: 10px; } + body.global-single-mode #global-section .field-label { width: 140px; font-size: 13px; } + body.global-single-mode #global-section select, + body.global-single-mode #global-section input[type=text] { padding: 6px 8px; font-size: 13px; } + body.global-single-mode #export-code { min-height: 180px; } @@ -107,17 +134,17 @@