From a224232bca2e2ead08eb9edc5fba871420161781 Mon Sep 17 00:00:00 2001 From: jpizarro Date: Thu, 19 Feb 2026 07:54:09 +0100 Subject: [PATCH 1/5] Add BAM extraction support and namespace repo mapping --- PLUGIN_RULES.md | 54 ++++++ api/repo_utils.py | 23 +-- api/settings.py | 67 ++++++- extractor/graph_builder.py | 163 ++++++++++++++--- extractor/tests/test_graph_builder_bam.py | 167 ++++++++++++++++++ extractor/tests/test_settings_repo_mapping.py | 76 ++++++++ 6 files changed, 493 insertions(+), 57 deletions(-) create mode 100644 PLUGIN_RULES.md create mode 100644 extractor/tests/test_graph_builder_bam.py create mode 100644 extractor/tests/test_settings_repo_mapping.py diff --git a/PLUGIN_RULES.md b/PLUGIN_RULES.md new file mode 100644 index 0000000..cc4d014 --- /dev/null +++ b/PLUGIN_RULES.md @@ -0,0 +1,54 @@ +# Schema Plugin Rules (Draft) + +This document defines the current extension contract used by Schema Studio to load and edit schema repositories. + +## 1) Profile Contract (Light Mode) +A schema profile must define: +- `key`: profile identifier (e.g. `nomad`, `bam`) +- `package_import`: top-level Python import package +- `package_dist`: pip distribution name +- `default_branch`: branch used by Light Mode +- `default_remote_repo`: canonical git URL used for `schema/update` +- `default_base_namespace`: namespace root for package discovery +- `default_package`: default module for graph extraction + +Current implementation lives in `api/light_mode/schema_source.py`. + +## 2) Namespace-to-Repo Mapping (Dev Mode) +A schema namespace is mapped to a git repository via: +- default mapping in `api/settings.py` +- optional env override: `SCHEMA_UML_REPO_MAP` + +Format: +- `SCHEMA_UML_REPO_MAP="ns.prefix=/path/or/url,other.prefix=/path/or/url"` + +Matching rule: +- longest namespace prefix wins. + +## 3) Extractor Entity Contract +The graph extractor (`extractor/graph_builder.py`) currently supports two shapes: + +### NOMAD-like entities +Class must expose at least one of: +- `m_def` +- `quantities` +- `sub_sections` + +### BAM-like entities +Class must inherit from BAM metadata base classes: +- `bam_masterdata.metadata.entities.ObjectType` / `CollectionType` / `DatasetType` +- `bam_masterdata.metadata.entities.VocabularyType` + +Quantities are extracted from: +- `PropertyTypeAssignment` class attributes (object/dataset/collection types) +- `VocabularyTerm` class attributes (vocabulary types) + +## 4) Editing Contract (Current) +Current CRUD in Schema Studio is graph-level and persisted as custom edits. +- persisted edits are replayed into graph responses +- this is not yet a full source-code writer/committer for schema files + +## 5) Planned Pluginization (TODO) +- Externalize extractor adapters into a registry (instead of built-in NOMAD/BAM checks). +- Define adapter hooks (section detection, quantity extraction, dtype/cardinality mapping). +- Add optional code-writer plugin hooks for repository write-back and commit/push workflow. diff --git a/api/repo_utils.py b/api/repo_utils.py index e4bae33..dcf03c3 100644 --- a/api/repo_utils.py +++ b/api/repo_utils.py @@ -13,7 +13,7 @@ def python_root(wt: Path) -> Path: """ Return the directory under which Python packages live in the worktree. - In many NOMAD schemas this is typically /src. + In many schema repos this is typically /src. If that does not exist, fall back to the worktree root. """ src = wt / "src" @@ -23,13 +23,6 @@ def python_root(wt: Path) -> Path: def list_modules_under(root: Path, base_package: str) -> List[str]: """ List all importable Python modules under the given base package. - - Example: - root = /.../worktrees/develop/src - base_package = "nomad_simulations.schema_packages" - → finds modules like: - "nomad_simulations.schema_packages.model_method", - "nomad_simulations.schema_packages.workflow.general", ... """ parts = base_package.split(".") pkg_dir = root.joinpath(*parts) @@ -39,13 +32,9 @@ def list_modules_under(root: Path, base_package: str) -> List[str]: modules: set[str] = set() for path in pkg_dir.rglob("*.py"): - # Compute module name relative to python root (src or repo root) rel = path.relative_to(root) - - # Strip .py suffix and convert path to dotted module path rel_no_ext = rel.with_suffix("") mod_name = ".".join(rel_no_ext.parts) - modules.add(mod_name) return sorted(modules) @@ -68,25 +57,19 @@ def bases_by_repo(base_packages: list[str]) -> dict[str, list[str]]: def primary_repo(package: str | None, base_namespace: str | None) -> str: """ Decide which repo should be used for a given package/base namespace. - - Mirrors the previous logic embedded in routes_git. """ if base_namespace: bases = parse_base_packages(base_namespace) if bases: return repo_for_base_namespace(bases[0]) + if package: - # Use the top-level namespace to infer the owning repository - prefix = package.split(".") - for i in range(len(prefix), 0, -1): - candidate = ".".join(prefix[:i]) - if candidate.startswith("nomad_measurements"): - return repo_for_base_namespace(candidate) return repo_for_base_namespace(package) defaults = parse_base_packages(DEFAULT_BASE_PACKAGE) if defaults: return repo_for_base_namespace(defaults[0]) + return repo_for_base_namespace(DEFAULT_BASE_PACKAGE) diff --git a/api/settings.py b/api/settings.py index c68435a..dfa43e0 100644 --- a/api/settings.py +++ b/api/settings.py @@ -7,7 +7,7 @@ DATA_DIR = Path(os.getenv("SCHEMA_UML_DATA_DIR", Path(__file__).resolve().parent / "_data")) DATA_DIR.mkdir(parents=True, exist_ok=True) -# Local path or remote URL to the schema repository (supports NOMAD-compatible schemas) +# Local path or remote URL to the primary schema repository SCHEMA_REPO = ( os.getenv("SCHEMA_UML_REPO") or os.getenv("NOMAD_SIM_REPO") @@ -15,8 +15,9 @@ or str(Path.home() / "src/nomad-simulations") ) -# Optional: secondary repo for nomad-measurements +# Optional secondary repositories keyed by namespace prefix MEASURE_REPO = os.getenv("NOMAD_MEASURE_REPO") or str(Path.home() / "src/nomad-measurements") +BAM_MASTERDATA_REPO = os.getenv("BAM_MASTERDATA_REPO") def _repo_slug(src: str) -> str: @@ -26,7 +27,6 @@ def _repo_slug(src: str) -> str: name = Path(path).name or "schema-repo" if name.endswith(".git"): name = name[:-4] - # Keep alnum + separators stable name = re.sub(r"[^A-Za-z0-9._-]", "_", name) return name or "schema-repo" @@ -36,7 +36,6 @@ def _repo_slug(src: str) -> str: # Default base package/section can be overridden per request or via env vars DEFAULT_BASE_PACKAGE = os.getenv( "SCHEMA_UML_BASE_PACKAGE", - # Default to the simulations schema; opt into nomad-measurements via env var "nomad_simulations.schema_packages", ) @@ -51,8 +50,20 @@ def _primary_base_package(base_packages: str) -> str: return "nomad_simulations.schema_packages" +def _default_package_for_base(base_package: str) -> str: + """Return a sensible default module for a base namespace.""" + + base = base_package.strip() + if base.startswith("bam_masterdata.datamodel"): + return "bam_masterdata.datamodel.object_types" + if base.startswith("bam_masterdata"): + return f"{base}.object_types" + return f"{base}.model_method" + + DEFAULT_PACKAGE = os.getenv( - "SCHEMA_UML_PACKAGE", f"{_primary_base_package(DEFAULT_BASE_PACKAGE)}.model_method" + "SCHEMA_UML_PACKAGE", + _default_package_for_base(_primary_base_package(DEFAULT_BASE_PACKAGE)), ) # Default branch for git operations @@ -65,10 +76,50 @@ def _primary_base_package(base_packages: str) -> str: MONGO_DB = os.getenv("SCHEMA_UML_MONGO_DB", "schema_uml") +def _parse_repo_map(raw: str | None) -> list[tuple[str, str]]: + """ + Parse SCHEMA_UML_REPO_MAP entries in the form: + "ns.prefix=/path/or/url,other.prefix=/path/or/url" + """ + + if not raw: + return [] + + pairs: list[tuple[str, str]] = [] + for item in raw.split(","): + chunk = item.strip() + if not chunk or "=" not in chunk: + continue + prefix, repo = chunk.split("=", 1) + prefix = prefix.strip() + repo = repo.strip() + if prefix and repo: + pairs.append((prefix, repo)) + pairs.sort(key=lambda p: len(p[0]), reverse=True) + return pairs + + +_NAMESPACE_REPO_DEFAULTS: list[tuple[str, str]] = [("nomad_measurements", MEASURE_REPO)] +if BAM_MASTERDATA_REPO: + _NAMESPACE_REPO_DEFAULTS.append(("bam_masterdata", BAM_MASTERDATA_REPO)) +_NAMESPACE_REPO_DEFAULTS = [(ns, repo) for ns, repo in _NAMESPACE_REPO_DEFAULTS if repo] +_NAMESPACE_REPO_DEFAULTS.sort(key=lambda p: len(p[0]), reverse=True) + +# TODO(plugin): replace env-based mapping with a first-class plugin manifest. +_NAMESPACE_REPO_OVERRIDES = _parse_repo_map(os.getenv("SCHEMA_UML_REPO_MAP")) + + def repo_for_base_namespace(base_package: str) -> str: """Return the repository source that owns the given base namespace.""" - base = base_package.strip() - if base.startswith("nomad_measurements"): - return MEASURE_REPO + base = (base_package or "").strip() + + for ns_prefix, repo in _NAMESPACE_REPO_OVERRIDES: + if base == ns_prefix or base.startswith(f"{ns_prefix}."): + return repo + + for ns_prefix, repo in _NAMESPACE_REPO_DEFAULTS: + if base == ns_prefix or base.startswith(f"{ns_prefix}."): + return repo + return SCHEMA_REPO diff --git a/extractor/graph_builder.py b/extractor/graph_builder.py index c3d2784..d798f99 100644 --- a/extractor/graph_builder.py +++ b/extractor/graph_builder.py @@ -38,18 +38,28 @@ def _normalize_doc(s: Any) -> Optional[str]: def _doc_from(obj: Any) -> Optional[str]: """ Try to extract a human doc/description from common attributes used by - NOMAD-style metainfo (description, m_def.description) and fall back to __doc__. + NOMAD-style metainfo and BAM masterdata definitions. """ for attr in ("description", "doc", "desc"): v = getattr(obj, attr, None) v = _normalize_doc(v) if v: return v + + # NOMAD sections definitions mdef = getattr(obj, "m_def", None) if mdef is not None: v = _normalize_doc(getattr(mdef, "description", None)) if v: return v + + # openBIS entities definitions + defs = getattr(obj, "defs", None) + if defs is not None: + v = _normalize_doc(getattr(defs, "description", None)) + if v: + return v + return _normalize_doc(getattr(obj, "__doc__", None)) @@ -59,7 +69,8 @@ def list_sections(package: str) -> List[str]: mod = importlib.import_module(package) base_ns = _root_namespace(package) return sorted( - name for name, obj in vars(mod).items() + name + for name, obj in vars(mod).items() if _is_section(obj) and _module_in_namespace(obj, base_ns) ) @@ -74,11 +85,14 @@ def build_graph( base_namespace: Optional[str] = None, exclude_prefixes: Tuple[str, ...] = ("nomad.metainfo.",), max_nodes: int = 8000, - max_depth: int = 20, # recurse reasonably deep + max_depth: int = 20, ) -> Dict[str, Any]: """ Build a graph starting at `root` (if given) or all section classes in `package`. - Properly resolves SubSection targets that are Section definitions, and recurses. + + Supports: + - NOMAD metainfo sections/quantities/subsections. + - BAM masterdata object/vocabulary classes and assigned properties/terms. """ mod = importlib.import_module(package) if base_namespace is None: @@ -115,10 +129,7 @@ def add_section(sec_obj: Any, depth: int = 0): if not _module_allowed(sec_mod, base_namespace, exclude_prefixes, allow_cross_module): return - # collect methods defined in your package methods = _public_methods(sec_obj, base_namespace) - - # robust doc extraction doc = _doc_from(sec_obj) nodes[sec_id] = Node( @@ -127,7 +138,7 @@ def add_section(sec_obj: Any, depth: int = 0): label=sec_name, doc=doc, module=sec_mod, - methods=methods or None + methods=methods or None, ) if len(nodes) > max_nodes: return @@ -140,12 +151,12 @@ def add_section(sec_obj: Any, depth: int = 0): id=qid, kind="quantity", label=qname, - doc=_doc_from(q), # ← include quantity doc + doc=_doc_from(q), dtype=_dtype_from(q), shape=_shape_from(q), card=_cardinality_from(q), owner=sec_id, - module=sec_mod + module=sec_mod, ) add_edge(Edge(source=sec_id, target=qid, type="hasQuantity", card=_cardinality_from(q))) if len(nodes) > max_nodes: @@ -157,6 +168,7 @@ def add_section(sec_obj: Any, depth: int = 0): continue if not _is_section(base): continue + base_mod = getattr(base, "__module__", "") if not _module_allowed(base_mod, base_namespace, exclude_prefixes, allow_cross_module): continue @@ -167,7 +179,7 @@ def add_section(sec_obj: Any, depth: int = 0): add_edge(Edge(source=sec_id, target=base_id, type="inherits")) if include_subsections: - for sname, s in _get_subsections(sec_obj): + for _, s in _get_subsections(sec_obj): tgt_cls = _resolve_section_class(_target_section_obj(s)) if tgt_cls is None: continue @@ -177,9 +189,7 @@ def add_section(sec_obj: Any, depth: int = 0): tgt_name = getattr(tgt_cls, "__name__", str(tgt_cls)) tgt_id = f"{tgt_mod}.{tgt_name}" - # add child section and quantities recursively add_section(tgt_cls, depth + 1) - # add UML composition edge add_edge(Edge(source=sec_id, target=tgt_id, type="hasSubSection", card=_cardinality_from(s))) if len(nodes) > max_nodes: return @@ -202,15 +212,56 @@ def add_section(sec_obj: Any, depth: int = 0): "root": root, "base_namespace": base_namespace, "nodes": [asdict(n) for n in node_list], - "edges": [asdict(e) for e in edges] + "edges": [asdict(e) for e in edges], } # -------- introspection helpers -------- -def _is_section(obj) -> bool: - # Section classes have m_def; be liberal to support variations - return inspect.isclass(obj) and (hasattr(obj, "m_def") or hasattr(obj, "quantities") or hasattr(obj, "sub_sections")) +def _is_section(obj: Any) -> bool: + return inspect.isclass(obj) and (_is_nomad_section(obj) or _is_bam_entity_class(obj)) + + +def _is_nomad_section(obj: Any) -> bool: + return hasattr(obj, "m_def") or hasattr(obj, "quantities") or hasattr(obj, "sub_sections") + + +def _is_bam_entity_class(obj: Any) -> bool: + for base in getattr(obj, "__mro__", []): + mod = getattr(base, "__module__", "") + if mod != "bam_masterdata.metadata.entities": + continue + if base.__name__ in {"ObjectType", "VocabularyType"}: + return True + return False + + +def _is_bam_object_type_class(obj: Any) -> bool: + for base in getattr(obj, "__mro__", []): + mod = getattr(base, "__module__", "") + if mod != "bam_masterdata.metadata.entities": + continue + if base.__name__ in {"ObjectType"}: + return True + return False + + +def _is_bam_vocabulary_type_class(obj: Any) -> bool: + for base in getattr(obj, "__mro__", []): + mod = getattr(base, "__module__", "") + if mod != "bam_masterdata.metadata.entities": + continue + if base.__name__ == "VocabularyType": + return True + return False + + +def _is_bam_property_assignment(obj: Any) -> bool: + return obj.__class__.__name__ == "PropertyTypeAssignment" and obj.__class__.__module__ == "bam_masterdata.metadata.definitions" + + +def _is_bam_vocabulary_term(obj: Any) -> bool: + return obj.__class__.__name__ == "VocabularyTerm" and obj.__class__.__module__ == "bam_masterdata.metadata.definitions" def _items_from_mapping_or_list(x) -> Iterable[Tuple[str, Any]]: @@ -242,6 +293,11 @@ def _cardinality_from(obj) -> Optional[str]: return f"{int(low)}..{hi}" except Exception: return str(c) + if hasattr(obj, "mandatory"): + try: + return "1..1" if bool(getattr(obj, "mandatory")) else "0..1" + except Exception: + pass return None @@ -255,7 +311,6 @@ def _name_from_target(target: Any) -> Optional[str]: if cls is None: return None - # Prefer the section/class name without a long module prefix name = getattr(cls, "__name__", None) or getattr(cls, "name", None) if not name: return None @@ -267,7 +322,6 @@ def _name_from_target(target: Any) -> Optional[str]: return f"{mod_short}.{name}" return name - # Common attributes exposed by NOMAD Reference dtypes for attr in ( "target_section_def", "target_section_cls", @@ -282,8 +336,34 @@ def _name_from_target(target: Any) -> Optional[str]: return None +def _enumish_value(value: Any) -> str: + enum_value = getattr(value, "value", None) + if isinstance(enum_value, str) and enum_value: + return enum_value + text = str(value) + if text.startswith("DataType."): + return text.split(".", 1)[1] + return text + + def _dtype_from(q) -> Optional[str]: - for attr in ("dtype", "type"): + if _is_bam_vocabulary_term(q): # Vocabulary terms are similar to enums, but we want to preserve the link to the vocabulary + return "VOCAB_TERM" + + if hasattr(q, "data_type"): # openBIS specific data types + try: + raw_dtype = _enumish_value(getattr(q, "data_type")) + vocabulary_code = getattr(q, "vocabulary_code", None) + object_code = getattr(q, "object_code", None) + if vocabulary_code: + return f"{raw_dtype}[{vocabulary_code}]" + if object_code: + return f"{raw_dtype}[{object_code}]" + return raw_dtype + except Exception: + pass + + for attr in ("dtype", "type"): # NOMAD specific data types if not hasattr(q, attr): continue try: @@ -306,40 +386,64 @@ def _shape_from(q) -> Optional[str]: return None +def _get_bam_quantities(sec_obj: Any) -> Iterable[Tuple[str, Any]]: + collected: Dict[str, Any] = {} + + if _is_bam_object_type_class(sec_obj): + for base in reversed(getattr(sec_obj, "__mro__", [])): + for attr_name, attr_value in getattr(base, "__dict__", {}).items(): + if _is_bam_property_assignment(attr_value): + collected[attr_name] = attr_value + return collected.items() + + if _is_bam_vocabulary_type_class(sec_obj): + for base in reversed(getattr(sec_obj, "__mro__", [])): + for attr_name, attr_value in getattr(base, "__dict__", {}).items(): + if _is_bam_vocabulary_term(attr_value): + collected[attr_name] = attr_value + return collected.items() + + return [] + + def _get_quantities(sec_obj) -> Iterable[Tuple[str, Any]]: - # Try class-level first qmap = getattr(sec_obj, "quantities", None) if qmap: return _items_from_mapping_or_list(qmap) - # Then via m_def reflection + mdef = getattr(sec_obj, "m_def", None) if mdef is not None: for attr in ("all_quantities", "quantities"): qmap = getattr(mdef, attr, None) if qmap: return _items_from_mapping_or_list(qmap) + + bam = _get_bam_quantities(sec_obj) + if bam: + return bam + return [] def _get_subsections(sec_obj) -> Iterable[Tuple[str, Any]]: - # Try class-level first smap = getattr(sec_obj, "sub_sections", None) if smap: return _items_from_mapping_or_list(smap) - # Then via m_def reflection + mdef = getattr(sec_obj, "m_def", None) if mdef is not None: for attr in ("all_sub_sections", "sub_sections"): smap = getattr(mdef, attr, None) if smap: return _items_from_mapping_or_list(smap) + return [] def _target_section_obj(subsec_obj): """ Return the raw target stored in SubSection definition. - It may be a *Section class* or a *Section definition* (metainfo Section). + It may be a section class or a section definition (metainfo section). """ for attr in ("section_def", "target", "sub_section", "section"): if hasattr(subsec_obj, attr): @@ -350,19 +454,19 @@ def _target_section_obj(subsec_obj): def _resolve_section_class(target) -> Optional[type]: """ If target is already a class, return it. - If it's a metainfo Section definition, try common attributes to reach the Python class. + If it's a metainfo section definition, try common attributes to reach the Python class. """ if target is None: return None if inspect.isclass(target): return target - # NOMAD metainfo Section definitions often expose the class as section_cls / section_class + for attr in ("section_cls", "section_class", "cls", "python_type"): if hasattr(target, attr): cand = getattr(target, attr) if inspect.isclass(cand): return cand - # Some definitions keep a 'm_root' or similar to the owning class; try heuristics + mod = getattr(target, "__module__", "") name = getattr(target, "__name__", None) or getattr(target, "name", None) if mod and name: @@ -380,6 +484,8 @@ def _root_namespace(package: str) -> str: parts = package.split(".") if len(parts) >= 3: return ".".join(parts[:3]) + if len(parts) == 2: + return ".".join(parts) return parts[0] @@ -388,7 +494,6 @@ def _module_allowed(module: str, base_namespace: str, exclude_prefixes: Tuple[st return False if allow_cross_module: return module.startswith(base_namespace) - # same module only if cross-mod disabled return module.startswith(base_namespace) diff --git a/extractor/tests/test_graph_builder_bam.py b/extractor/tests/test_graph_builder_bam.py new file mode 100644 index 0000000..07dbfaf --- /dev/null +++ b/extractor/tests/test_graph_builder_bam.py @@ -0,0 +1,167 @@ +from __future__ import annotations + +from pathlib import Path +import importlib +import sys + + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +from extractor.graph_builder import build_graph, list_sections + + +def _write_fake_bam_package(root: Path) -> None: + files: dict[str, str] = { + "bam_masterdata/__init__.py": "", + "bam_masterdata/metadata/__init__.py": "", + "bam_masterdata/datamodel/__init__.py": "", + "bam_masterdata/metadata/entities.py": """ +class ObjectType: + pass + + +class CollectionType(ObjectType): + pass + + +class DatasetType(ObjectType): + pass + + +class VocabularyType: + pass +""", + "bam_masterdata/metadata/definitions.py": """ +from dataclasses import dataclass + + +class DataType: + def __init__(self, value: str): + self.value = value + + def __str__(self) -> str: + return f"DataType.{self.value}" + + +@dataclass +class PropertyTypeAssignment: + code: str + data_type: object + property_label: str + description: str + mandatory: bool + vocabulary_code: str | None = None + object_code: str | None = None + + +@dataclass +class VocabularyTerm: + code: str + label: str + description: str +""", + "bam_masterdata/datamodel/object_types.py": """ +from bam_masterdata.metadata.definitions import DataType, PropertyTypeAssignment +from bam_masterdata.metadata.entities import ObjectType + + +class BaseEntity(ObjectType): + base_value = PropertyTypeAssignment( + code="BASE_VALUE", + data_type=DataType("INTEGER"), + property_label="Base value", + description="Base value", + mandatory=False, + ) + + +class Device(BaseEntity): + identifier = PropertyTypeAssignment( + code="IDENTIFIER", + data_type=DataType("VARCHAR"), + property_label="Identifier", + description="Unique identifier", + mandatory=True, + ) + + status = PropertyTypeAssignment( + code="STATUS", + data_type=DataType("CONTROLLEDVOCABULARY"), + property_label="Status", + description="Device status", + mandatory=False, + vocabulary_code="DEVICE_STATUS", + ) +""", + "bam_masterdata/datamodel/vocabulary_types.py": """ +from bam_masterdata.metadata.definitions import VocabularyTerm +from bam_masterdata.metadata.entities import VocabularyType + + +class DeviceStatus(VocabularyType): + active = VocabularyTerm(code="ACTIVE", label="Active", description="Device is active") +""", + } + + for rel, content in files.items(): + path = root / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +def test_build_graph_extracts_bam_object_types_and_controlled_vocab(monkeypatch, tmp_path: Path): + _write_fake_bam_package(tmp_path) + monkeypatch.syspath_prepend(str(tmp_path)) + importlib.invalidate_caches() + + sections = list_sections("bam_masterdata.datamodel.object_types") + assert sections == ["BaseEntity", "Device"] + + graph = build_graph( + package="bam_masterdata.datamodel.object_types", + root="Device", + base_namespace="bam_masterdata.datamodel", + ) + + nodes_by_id = {node["id"]: node for node in graph["nodes"]} + + section_id = "bam_masterdata.datamodel.object_types.Device" + assert section_id in nodes_by_id + + identifier = nodes_by_id[f"{section_id}.identifier"] + assert identifier["dtype"] == "VARCHAR" + assert identifier["card"] == "1..1" + + status = nodes_by_id[f"{section_id}.status"] + assert status["dtype"] == "CONTROLLEDVOCABULARY[DEVICE_STATUS]" + assert status["card"] == "0..1" + + inherited = nodes_by_id[f"{section_id}.base_value"] + assert inherited["dtype"] == "INTEGER" + + edge_types = {(edge["source"], edge["target"], edge["type"]) for edge in graph["edges"]} + assert ( + "bam_masterdata.datamodel.object_types.Device", + "bam_masterdata.datamodel.object_types.BaseEntity", + "inherits", + ) in edge_types + + +def test_build_graph_extracts_bam_vocabulary_terms(monkeypatch, tmp_path: Path): + _write_fake_bam_package(tmp_path) + monkeypatch.syspath_prepend(str(tmp_path)) + importlib.invalidate_caches() + + graph = build_graph( + package="bam_masterdata.datamodel.vocabulary_types", + root="DeviceStatus", + base_namespace="bam_masterdata.datamodel", + ) + + nodes_by_id = {node["id"]: node for node in graph["nodes"]} + term_id = "bam_masterdata.datamodel.vocabulary_types.DeviceStatus.active" + assert term_id in nodes_by_id + assert nodes_by_id[term_id]["dtype"] == "VOCAB_TERM" + assert nodes_by_id[term_id]["doc"] == "Device is active" diff --git a/extractor/tests/test_settings_repo_mapping.py b/extractor/tests/test_settings_repo_mapping.py new file mode 100644 index 0000000..58a12a6 --- /dev/null +++ b/extractor/tests/test_settings_repo_mapping.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +from pathlib import Path +import importlib +import sys + + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + + +def _reload_settings(monkeypatch, **env): + keys = { + "SCHEMA_UML_REPO", + "NOMAD_SIM_REPO", + "GIT_REPO_DIR", + "NOMAD_MEASURE_REPO", + "BAM_MASTERDATA_REPO", + "SCHEMA_UML_REPO_MAP", + "SCHEMA_UML_BASE_PACKAGE", + "SCHEMA_UML_PACKAGE", + } + for key in keys: + monkeypatch.delenv(key, raising=False) + + for key, value in env.items(): + monkeypatch.setenv(key, value) + + sys.modules.pop("api.settings", None) + import api.settings as settings_mod + + return importlib.reload(settings_mod) + + +def test_bam_repo_is_selected_by_namespace(monkeypatch, tmp_path: Path): + default_repo = tmp_path / "default" + bam_repo = tmp_path / "bam" + + mod = _reload_settings( + monkeypatch, + SCHEMA_UML_REPO=str(default_repo), + BAM_MASTERDATA_REPO=str(bam_repo), + ) + + assert mod.repo_for_base_namespace("bam_masterdata.datamodel") == str(bam_repo) + assert mod.repo_for_base_namespace("bam_masterdata.datamodel.object_types") == str(bam_repo) + assert mod.repo_for_base_namespace("nomad_simulations.schema_packages") == str(default_repo) + + +def test_repo_map_overrides_default_namespace_mapping(monkeypatch, tmp_path: Path): + default_repo = tmp_path / "default" + bam_repo = tmp_path / "bam" + mapped_repo = tmp_path / "mapped" + + mod = _reload_settings( + monkeypatch, + SCHEMA_UML_REPO=str(default_repo), + BAM_MASTERDATA_REPO=str(bam_repo), + SCHEMA_UML_REPO_MAP=f"bam_masterdata.datamodel={mapped_repo}", + ) + + assert mod.repo_for_base_namespace("bam_masterdata.datamodel") == str(mapped_repo) + assert mod.repo_for_base_namespace("bam_masterdata.datamodel.vocabulary_types") == str(mapped_repo) + + +def test_default_package_for_bam_namespace(monkeypatch, tmp_path: Path): + default_repo = tmp_path / "default" + + mod = _reload_settings( + monkeypatch, + SCHEMA_UML_REPO=str(default_repo), + SCHEMA_UML_BASE_PACKAGE="bam_masterdata.datamodel", + ) + + assert mod.DEFAULT_PACKAGE == "bam_masterdata.datamodel.object_types" From b32f64e5462d11b2072a671522347bbdf8d7d31e Mon Sep 17 00:00:00 2001 From: jpizarro Date: Thu, 19 Feb 2026 07:54:18 +0100 Subject: [PATCH 2/5] Add Light Mode schema profiles for NOMAD and BAM --- api/light_mode/app.py | 10 +- api/light_mode/schema_source.py | 102 ++++++++++++++---- .../tests/test_schema_source_profiles.py | 52 +++++++++ 3 files changed, 140 insertions(+), 24 deletions(-) create mode 100644 api/light_mode/tests/test_schema_source_profiles.py diff --git a/api/light_mode/app.py b/api/light_mode/app.py index f897ccf..e7b04d3 100644 --- a/api/light_mode/app.py +++ b/api/light_mode/app.py @@ -23,7 +23,10 @@ from extractor.graph_builder import _root_namespace, build_graph, list_sections from extractor.usage_index import get_usage_for_section from .schema_source import ( + DEFAULT_BASE_NAMESPACE as LIGHT_DEFAULT_BASE_NS, DEFAULT_BRANCH as LIGHT_MODE_BRANCH, + DEFAULT_PACKAGE as LIGHT_DEFAULT_PACKAGE, + LIGHT_PROFILE_KEY, SchemaUnavailable, current_schema_info, list_modules_for_base, @@ -36,8 +39,8 @@ SEND_ENDPOINT = os.getenv("SCHEMA_STUDIO_SEND_ENDPOINT") DEFAULT_PORT = int(os.getenv("SCHEMA_STUDIO_PORT", "5179")) DEFAULT_HOST = os.getenv("SCHEMA_STUDIO_HOST", "127.0.0.1") -DEFAULT_PACKAGE = os.getenv("SCHEMA_STUDIO_DEFAULT_PACKAGE", "nomad_simulations.schema_packages.model_method") -DEFAULT_BASE_NS = os.getenv("SCHEMA_STUDIO_DEFAULT_NAMESPACE", "nomad_simulations.schema_packages") +DEFAULT_PACKAGE = os.getenv("SCHEMA_STUDIO_DEFAULT_PACKAGE", LIGHT_DEFAULT_PACKAGE) +DEFAULT_BASE_NS = os.getenv("SCHEMA_STUDIO_DEFAULT_NAMESPACE", LIGHT_DEFAULT_BASE_NS) AUTO_BOOTSTRAP_SCHEMA = os.getenv("SCHEMA_STUDIO_AUTO_BOOTSTRAP_SCHEMA", "1").lower() not in {"0", "false", "no"} logger = logging.getLogger(__name__) @@ -424,7 +427,7 @@ async def _bootstrap_schema_on_startup(): @app.get("/schema/version") async def schema_version(): info = _schema_info_or_503(auto_bootstrap=AUTO_BOOTSTRAP_SCHEMA) - return {"version": info.version, "source": info.source, "send_design_enabled": bool(SEND_ENDPOINT)} + return {"version": info.version, "source": info.source, "schema_profile": LIGHT_PROFILE_KEY, "send_design_enabled": bool(SEND_ENDPOINT)} @app.post("/schema/update") @@ -445,6 +448,7 @@ async def health(): "workspace": _workspace_payload(_workspace()), "schema_version": info.version, "schema_source": info.source, + "schema_profile": LIGHT_PROFILE_KEY, "send_design_enabled": bool(SEND_ENDPOINT), } diff --git a/api/light_mode/schema_source.py b/api/light_mode/schema_source.py index 07a8c3e..300ab66 100644 --- a/api/light_mode/schema_source.py +++ b/api/light_mode/schema_source.py @@ -1,8 +1,8 @@ """Schema sourcing for Light Mode. Policy: -- Always use the installed `nomad-simulations` Python package. -- Always target the remote `develop` branch lineage for updates. +- Always use an installed schema package selected by profile. +- Light mode branch is fixed by profile (e.g. develop/main). - Never use local checkouts/worktrees. """ from __future__ import annotations @@ -11,15 +11,71 @@ import importlib.metadata import importlib.util import json +import os import subprocess import sys from dataclasses import dataclass from pathlib import Path -PACKAGE_IMPORT = "nomad_simulations" -PACKAGE_DIST = "nomad-simulations" -DEFAULT_BRANCH = "develop" -DEFAULT_REMOTE_REPO = "https://github.com/nomad-coe/nomad-simulations.git" + +@dataclass(frozen=True) +class SchemaProfile: + key: str + package_import: str + package_dist: str + default_branch: str + default_remote_repo: str + default_base_namespace: str + default_package: str + + +SCHEMA_PROFILES: dict[str, SchemaProfile] = { + "nomad": SchemaProfile( + key="nomad", + package_import="nomad_simulations", + package_dist="nomad-simulations", + default_branch="develop", + default_remote_repo="https://github.com/nomad-coe/nomad-simulations.git", + default_base_namespace="nomad_simulations.schema_packages", + default_package="nomad_simulations.schema_packages.model_method", + ), + "bam": SchemaProfile( + key="bam", + package_import="bam_masterdata", + package_dist="bam-masterdata", + default_branch="main", + default_remote_repo="https://github.com/BAMresearch/bam-masterdata.git", + default_base_namespace="bam_masterdata.datamodel", + default_package="bam_masterdata.datamodel.object_types", + ), +} + + +def _select_profile() -> SchemaProfile: + requested = os.getenv("SCHEMA_STUDIO_LIGHT_SCHEMA_PROFILE", "").strip().lower() + if requested in SCHEMA_PROFILES: + return SCHEMA_PROFILES[requested] + + if requested: + for profile in SCHEMA_PROFILES.values(): + if requested in {profile.package_import, profile.package_dist}: + return profile + + package_hint = os.getenv("SCHEMA_STUDIO_DEFAULT_PACKAGE", "") + if package_hint.startswith("bam_masterdata"): + return SCHEMA_PROFILES["bam"] + + return SCHEMA_PROFILES["nomad"] + + +ACTIVE_PROFILE = _select_profile() +LIGHT_PROFILE_KEY = ACTIVE_PROFILE.key +PACKAGE_IMPORT = ACTIVE_PROFILE.package_import +PACKAGE_DIST = ACTIVE_PROFILE.package_dist +DEFAULT_BRANCH = ACTIVE_PROFILE.default_branch +DEFAULT_REMOTE_REPO = ACTIVE_PROFILE.default_remote_repo +DEFAULT_BASE_NAMESPACE = ACTIVE_PROFILE.default_base_namespace +DEFAULT_PACKAGE = ACTIVE_PROFILE.default_package UPGRADE_TARGET = f"git+{DEFAULT_REMOTE_REPO}@{DEFAULT_BRANCH}" @@ -27,19 +83,23 @@ class SchemaInfo: package_root: Path version: str - source: str # "installed" | "remote-develop" + source: str # "installed" | "remote-" class SchemaUnavailable(RuntimeError): pass +def active_profile() -> SchemaProfile: + return ACTIVE_PROFILE + + def _distribution() -> importlib.metadata.Distribution: try: return importlib.metadata.distribution(PACKAGE_DIST) except importlib.metadata.PackageNotFoundError as exc: raise SchemaUnavailable( - "nomad-simulations is not installed. Reinstall Light Mode so it pulls the remote develop package." + f"{PACKAGE_DIST} is not installed. Reinstall Light Mode to pull the configured remote package." ) from exc @@ -47,14 +107,14 @@ def _package_root() -> Path: spec = importlib.util.find_spec(PACKAGE_IMPORT) if spec is None: raise SchemaUnavailable( - "Could not import nomad_simulations. Reinstall Light Mode to restore schema package availability." + f"Could not import {PACKAGE_IMPORT}. Reinstall Light Mode to restore schema package availability." ) location = spec.submodule_search_locations if location: return Path(next(iter(location))).resolve() if spec.origin: return Path(spec.origin).resolve().parent - raise SchemaUnavailable("Could not determine nomad_simulations install location.") + raise SchemaUnavailable(f"Could not determine install location for {PACKAGE_IMPORT}.") def _direct_url_payload(dist: importlib.metadata.Distribution) -> dict | None: @@ -71,6 +131,11 @@ def _direct_url_payload(dist: importlib.metadata.Distribution) -> dict | None: return parsed if isinstance(parsed, dict) else None +def _normalize_repo(url: str) -> str: + base = url.rstrip("/") + return base[:-4] if base.endswith(".git") else base + + def _schema_info_from_install() -> SchemaInfo: dist = _distribution() package_root = _package_root() @@ -79,20 +144,15 @@ def _schema_info_from_install() -> SchemaInfo: direct_url = _direct_url_payload(dist) if not direct_url: - # Regular index/wheel installs do not include direct_url.json. return SchemaInfo(package_root=package_root, version=dist.version, source="installed") source_url = direct_url.get("url") if not isinstance(source_url, str) or source_url.startswith("file://"): - raise SchemaUnavailable("Light Mode does not support local nomad-simulations sources.") - - def normalize_repo(url: str) -> str: - base = url.rstrip("/") - return base[:-4] if base.endswith(".git") else base + raise SchemaUnavailable(f"Light Mode does not support local {PACKAGE_DIST} sources.") - if normalize_repo(source_url) != normalize_repo(DEFAULT_REMOTE_REPO): + if _normalize_repo(source_url) != _normalize_repo(DEFAULT_REMOTE_REPO): raise SchemaUnavailable( - "Light Mode must use the remote nomad-simulations repository on develop." + f"Light Mode for profile '{LIGHT_PROFILE_KEY}' must use repository {DEFAULT_REMOTE_REPO}." ) vcs_info = direct_url.get("vcs_info") @@ -102,12 +162,12 @@ def normalize_repo(url: str) -> str: requested = vcs_info.get("requested_revision") if requested and requested != DEFAULT_BRANCH: raise SchemaUnavailable( - f"Light Mode is pinned to remote {DEFAULT_BRANCH}; found installed revision {requested!r}." + f"Light Mode profile '{LIGHT_PROFILE_KEY}' is pinned to remote {DEFAULT_BRANCH}; found revision {requested!r}." ) commit = vcs_info.get("commit_id") version = commit if isinstance(commit, str) and commit else dist.version - return SchemaInfo(package_root=package_root, version=version, source="remote-develop") + return SchemaInfo(package_root=package_root, version=version, source=f"remote-{DEFAULT_BRANCH}") def ensure_schema_ready() -> SchemaInfo: @@ -122,7 +182,7 @@ def current_schema_info() -> SchemaInfo: def update_schema() -> SchemaInfo: """ - Upgrade to the latest develop branch package using pip. + Upgrade to the latest branch package for the active profile using pip. Keeps local Light Mode edits in SQLite. """ cmd = [sys.executable, "-m", "pip", "install", "--upgrade", UPGRADE_TARGET] diff --git a/api/light_mode/tests/test_schema_source_profiles.py b/api/light_mode/tests/test_schema_source_profiles.py new file mode 100644 index 0000000..a9cd107 --- /dev/null +++ b/api/light_mode/tests/test_schema_source_profiles.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import importlib +import sys +from pathlib import Path + + +PROJECT_ROOT = Path(__file__).resolve().parents[3] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + + +def _reload_schema_source(monkeypatch, *, profile: str | None = None, package_hint: str | None = None): + monkeypatch.delenv("SCHEMA_STUDIO_LIGHT_SCHEMA_PROFILE", raising=False) + monkeypatch.delenv("SCHEMA_STUDIO_DEFAULT_PACKAGE", raising=False) + + if profile is not None: + monkeypatch.setenv("SCHEMA_STUDIO_LIGHT_SCHEMA_PROFILE", profile) + if package_hint is not None: + monkeypatch.setenv("SCHEMA_STUDIO_DEFAULT_PACKAGE", package_hint) + + sys.modules.pop("api.light_mode.schema_source", None) + import api.light_mode.schema_source as mod + + return importlib.reload(mod) + + +def test_default_profile_is_nomad(monkeypatch): + mod = _reload_schema_source(monkeypatch) + + assert mod.LIGHT_PROFILE_KEY == "nomad" + assert mod.DEFAULT_BRANCH == "develop" + assert mod.DEFAULT_PACKAGE == "nomad_simulations.schema_packages.model_method" + assert mod.DEFAULT_BASE_NAMESPACE == "nomad_simulations.schema_packages" + + +def test_explicit_bam_profile(monkeypatch): + mod = _reload_schema_source(monkeypatch, profile="bam") + + assert mod.LIGHT_PROFILE_KEY == "bam" + assert mod.DEFAULT_BRANCH == "main" + assert mod.DEFAULT_PACKAGE == "bam_masterdata.datamodel.object_types" + assert mod.DEFAULT_BASE_NAMESPACE == "bam_masterdata.datamodel" + assert mod.UPGRADE_TARGET.endswith("bam-masterdata.git@main") + + +def test_package_hint_selects_bam_profile(monkeypatch): + mod = _reload_schema_source(monkeypatch, package_hint="bam_masterdata.datamodel.vocabulary_types") + + assert mod.LIGHT_PROFILE_KEY == "bam" + assert mod.PACKAGE_IMPORT == "bam_masterdata" + assert mod.PACKAGE_DIST == "bam-masterdata" From 9932f052e17d0c833052cc3a74372325f9eb2a62 Mon Sep 17 00:00:00 2001 From: jpizarro Date: Thu, 19 Feb 2026 07:54:24 +0100 Subject: [PATCH 3/5] Update frontend defaults and presets for BAM packages --- web/src/App.tsx | 6 +++--- web/src/api.ts | 4 ++-- web/src/components/OverviewGrid.tsx | 9 ++------- web/src/components/OverviewList.tsx | 8 ++------ web/src/constants/defaults.ts | 12 +++++++++++- 5 files changed, 20 insertions(+), 19 deletions(-) diff --git a/web/src/App.tsx b/web/src/App.tsx index ae72d0b..7caa5e8 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -112,7 +112,7 @@ export default function App() { const [theme, setTheme] = useState<"dark" | "light">(() => { if (typeof window === "undefined") return "dark"; const stored = window.localStorage.getItem("schema-uml-theme"); - const initial = stored === "light" ? "light" : "dark"; + const initial = stored === "light" ? "light" : (LIGHT_MODE ? "light" : "dark"); document.documentElement.setAttribute("data-theme", initial); return initial; }); @@ -2304,7 +2304,7 @@ export default function App() {

{isLightMode ? "Running in Light Mode (local, single-user, non-production)." - : "Craft diagrams, compare branches, and edit schemas. Currently defaults to nomad-simulations."} + : "Craft diagrams, compare branches, and edit schemas across compatible repositories."}

{loading || diffLoading ? "Working…" : "Ready"} @@ -2789,7 +2789,7 @@ export default function App() {
- 1) Go to and pick a root, then hit “Build graph” to load the nomad-simulations schema. + 1) Go to and pick a root, then hit “Build graph” to load your selected schema package.
2) See the panel to read class/quantity details as you browse. diff --git a/web/src/api.ts b/web/src/api.ts index bc18e60..a239331 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -1,8 +1,8 @@ import { API_FEATURE_HEADER, API_VERSION, API_VERSION_HEADER, DEFAULT_FEATURE_FLAGS } from "./constants/api"; +import { DEFAULT_PACKAGE } from "./constants/defaults"; import { ensureDiffResponse, type DiffResponse } from "./types/api"; const BASE = import.meta.env.VITE_API_BASE ?? "http://localhost:5179"; -const DEFAULT_PACKAGE = import.meta.env.VITE_DEFAULT_PACKAGE ?? "nomad_simulations.schema_packages.model_method"; const authHeaders = (): Record => { const base: Record = { @@ -28,7 +28,7 @@ export async function getDiff(base: string, head: string, pkg = DEFAULT_PACKAGE) const r = await fetch(`${BASE}/graph/diff`, { method: "POST", headers: { "content-type": "application/json", ...authHeaders() }, - body: JSON.stringify({ base, head, package: pkg }) + body: JSON.stringify({ base, head, package: pkg }), }); if (!r.ok) throw new Error(await r.text()); return ensureDiffResponse(await r.json()); diff --git a/web/src/components/OverviewGrid.tsx b/web/src/components/OverviewGrid.tsx index eeae97a..9d63d99 100644 --- a/web/src/components/OverviewGrid.tsx +++ b/web/src/components/OverviewGrid.tsx @@ -1,16 +1,13 @@ import { useEffect, useMemo, useState } from "react"; +import { DEFAULT_OVERVIEW_NAMESPACE } from "../constants/defaults"; type OverviewItem = { package: string; classes: string[] }; type OverviewResp = { branch: string; base: string; items: OverviewItem[] }; -const DEFAULT_OVERVIEW_BASE = - import.meta.env.VITE_DEFAULT_NAMESPACE ?? - "nomad_simulations.schema_packages,nomad_measurements"; - export default function OverviewGrid({ apiBase, branch, - base = DEFAULT_OVERVIEW_BASE, + base = DEFAULT_OVERVIEW_NAMESPACE, token, onClassSelect, }: { @@ -86,7 +83,6 @@ export default function OverviewGrid({
- {/* responsive grid */}
- {/* class chips */}
{ if (!r.ok) { diff --git a/web/src/constants/defaults.ts b/web/src/constants/defaults.ts index c2e83df..2da5d89 100644 --- a/web/src/constants/defaults.ts +++ b/web/src/constants/defaults.ts @@ -3,9 +3,12 @@ export const DEFAULT_PACKAGE = import.meta.env.VITE_DEFAULT_PACKAGE ?? "nomad_si export const DEFAULT_NAMESPACE = import.meta.env.VITE_DEFAULT_NAMESPACE ?? "nomad_simulations.schema_packages"; +export const DEFAULT_OVERVIEW_NAMESPACE = + import.meta.env.VITE_DEFAULT_OVERVIEW_NAMESPACE ?? + "nomad_simulations.schema_packages,nomad_measurements,bam_masterdata.datamodel"; export const DEFAULT_ROOT = import.meta.env.VITE_DEFAULT_ROOT ?? "ModelMethod"; export const LIGHT_MODE = (import.meta.env.VITE_LIGHT_MODE ?? "false").toLowerCase() === "true"; -export const DEFAULT_BRANCH = LIGHT_MODE ? "develop" : (import.meta.env.VITE_DEFAULT_BRANCH ?? "develop"); +export const DEFAULT_BRANCH = import.meta.env.VITE_DEFAULT_BRANCH ?? "develop"; export const WORKSPACE_PRESETS = [ { @@ -15,4 +18,11 @@ export const WORKSPACE_PRESETS = [ pkg: "nomad_simulations.schema_packages.model_method", root: "ModelMethod", }, + { + label: "bam-masterdata", + namespace: "bam_masterdata.datamodel", + branch: "main", + pkg: "bam_masterdata.datamodel.object_types", + root: "SearchQuery", + }, ]; From 80d4bb382bb93a4a7a2ffca9a572512bdd45d26a Mon Sep 17 00:00:00 2001 From: jpizarro Date: Thu, 19 Feb 2026 07:54:31 +0100 Subject: [PATCH 4/5] Load .env in CLI and add bam-masterdata dependency --- api/light_mode/cli.py | 89 ++++++++++++++++++++++++++++++++++++++++--- pyproject.toml | 1 + 2 files changed, 85 insertions(+), 5 deletions(-) diff --git a/api/light_mode/cli.py b/api/light_mode/cli.py index 1f12994..85e80f0 100644 --- a/api/light_mode/cli.py +++ b/api/light_mode/cli.py @@ -1,16 +1,86 @@ """CLI entrypoint for Schema Studio Light Mode.""" from __future__ import annotations +import os +from pathlib import Path import threading import time from urllib.request import urlopen -import os import webbrowser + import uvicorn -from .app import app, DEFAULT_HOST, DEFAULT_PORT -BANNER = "Running in Light Mode (local, single-user, non-production; schema pinned to nomad-simulations/develop)" +def _strip_wrapping_quotes(value: str) -> str: + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in {'"', "'"}: + return value[1:-1] + return value + + +def _load_env_fallback(path: Path) -> bool: + """Minimal .env loader used when python-dotenv is unavailable.""" + loaded_any = False + for raw_line in path.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line or line.startswith("#"): + continue + if line.startswith("export "): + line = line[len("export "):].strip() + if "=" not in line: + continue + key, value = line.split("=", 1) + key = key.strip() + if not key or key in os.environ: + continue + os.environ[key] = _strip_wrapping_quotes(value) + loaded_any = True + return loaded_any + + +def _load_env_file(path: Path) -> bool: + try: + from dotenv import load_dotenv # type: ignore + except Exception: + return _load_env_fallback(path) + return bool(load_dotenv(dotenv_path=path, override=False)) + + +def _candidate_env_files() -> list[Path]: + candidates: list[Path] = [] + + explicit = os.getenv("SCHEMA_STUDIO_ENV_FILE") + if explicit: + candidates.append(Path(explicit).expanduser()) + + candidates.append(Path.cwd() / ".env") + candidates.append(Path(__file__).resolve().parents[2] / ".env") + + unique: list[Path] = [] + seen: set[str] = set() + for p in candidates: + key = str(p) + if key in seen: + continue + seen.add(key) + unique.append(p) + return unique + + +def _load_first_available_env() -> Path | None: + for env_path in _candidate_env_files(): + if not env_path.is_file(): + continue + if _load_env_file(env_path): + return env_path + return None + + +def _banner(profile_key: str, branch: str) -> str: + return ( + "Running in Light Mode (local, single-user, non-production; " + f"schema profile={profile_key}, branch={branch})" + ) def _open_browser_when_ready(url: str, timeout_seconds: float = 120.0) -> None: @@ -27,13 +97,22 @@ def _open_browser_when_ready(url: str, timeout_seconds: float = 120.0) -> None: return except Exception: time.sleep(0.2) - # Fallback: open anyway if readiness check timed out. webbrowser.open(url) def main() -> None: + loaded_env = _load_first_available_env() + + # Import after loading .env so app defaults resolve from environment when present. + from .app import app, DEFAULT_HOST, DEFAULT_PORT + from .schema_source import active_profile + + profile = active_profile() url = f"http://{DEFAULT_HOST}:{DEFAULT_PORT}" - print(BANNER) + + if loaded_env is not None: + print(f"Loaded environment from {loaded_env}") + print(_banner(profile.key, profile.default_branch)) print(f"Launching browser at {url} when server is ready\n") threading.Thread(target=_open_browser_when_ready, args=(url,), daemon=True).start() diff --git a/pyproject.toml b/pyproject.toml index 197e13e..f492c42 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ dependencies = [ "httpx", "platformdirs", "nomad-simulations", + "bam-masterdata", ] [project.optional-dependencies] From d2a20c3904afbcd129853c6740a9e20d284654e9 Mon Sep 17 00:00:00 2001 From: jpizarro Date: Thu, 19 Feb 2026 08:04:39 +0100 Subject: [PATCH 5/5] Add docstrings --- api/light_mode/cli.py | 6 +++++ api/light_mode/schema_source.py | 13 +++++++++++ .../tests/test_schema_source_profiles.py | 4 ++++ api/repo_utils.py | 1 + api/settings.py | 1 + extractor/graph_builder.py | 23 +++++++++++++++++++ extractor/tests/test_graph_builder_bam.py | 3 +++ extractor/tests/test_settings_repo_mapping.py | 4 ++++ 8 files changed, 55 insertions(+) diff --git a/api/light_mode/cli.py b/api/light_mode/cli.py index 85e80f0..e09611d 100644 --- a/api/light_mode/cli.py +++ b/api/light_mode/cli.py @@ -12,6 +12,7 @@ def _strip_wrapping_quotes(value: str) -> str: + """Remove one matching pair of leading/trailing quotes from an env value.""" value = value.strip() if len(value) >= 2 and value[0] == value[-1] and value[0] in {'"', "'"}: return value[1:-1] @@ -39,6 +40,7 @@ def _load_env_fallback(path: Path) -> bool: def _load_env_file(path: Path) -> bool: + """Load a dotenv file with python-dotenv when available, else use fallback parser.""" try: from dotenv import load_dotenv # type: ignore except Exception: @@ -47,6 +49,7 @@ def _load_env_file(path: Path) -> bool: def _candidate_env_files() -> list[Path]: + """Return candidate `.env` files in precedence order, de-duplicated.""" candidates: list[Path] = [] explicit = os.getenv("SCHEMA_STUDIO_ENV_FILE") @@ -68,6 +71,7 @@ def _candidate_env_files() -> list[Path]: def _load_first_available_env() -> Path | None: + """Load the first existing env file that sets at least one variable.""" for env_path in _candidate_env_files(): if not env_path.is_file(): continue @@ -77,6 +81,7 @@ def _load_first_available_env() -> Path | None: def _banner(profile_key: str, branch: str) -> str: + """Build a startup banner describing active light-mode schema profile settings.""" return ( "Running in Light Mode (local, single-user, non-production; " f"schema profile={profile_key}, branch={branch})" @@ -101,6 +106,7 @@ def _open_browser_when_ready(url: str, timeout_seconds: float = 120.0) -> None: def main() -> None: + """Run the light-mode API server and open the UI once healthcheck responds.""" loaded_env = _load_first_available_env() # Import after loading .env so app defaults resolve from environment when present. diff --git a/api/light_mode/schema_source.py b/api/light_mode/schema_source.py index 300ab66..1ad8b30 100644 --- a/api/light_mode/schema_source.py +++ b/api/light_mode/schema_source.py @@ -20,6 +20,7 @@ @dataclass(frozen=True) class SchemaProfile: + """Profile configuration describing one supported schema source package.""" key: str package_import: str package_dist: str @@ -52,11 +53,13 @@ class SchemaProfile: def _select_profile() -> SchemaProfile: + """Select active schema profile from env override, then package hint, then default.""" requested = os.getenv("SCHEMA_STUDIO_LIGHT_SCHEMA_PROFILE", "").strip().lower() if requested in SCHEMA_PROFILES: return SCHEMA_PROFILES[requested] if requested: + # Allow values like "bam_masterdata" or "bam-masterdata" besides short keys. for profile in SCHEMA_PROFILES.values(): if requested in {profile.package_import, profile.package_dist}: return profile @@ -81,20 +84,24 @@ def _select_profile() -> SchemaProfile: @dataclass class SchemaInfo: + """Runtime metadata about the installed schema package source/version.""" package_root: Path version: str source: str # "installed" | "remote-" class SchemaUnavailable(RuntimeError): + """Raised when active schema profile cannot be validated or updated.""" pass def active_profile() -> SchemaProfile: + """Return the resolved schema profile for the current process.""" return ACTIVE_PROFILE def _distribution() -> importlib.metadata.Distribution: + """Return installed package distribution metadata for the active schema profile.""" try: return importlib.metadata.distribution(PACKAGE_DIST) except importlib.metadata.PackageNotFoundError as exc: @@ -104,6 +111,7 @@ def _distribution() -> importlib.metadata.Distribution: def _package_root() -> Path: + """Resolve filesystem location of the installed schema package.""" spec = importlib.util.find_spec(PACKAGE_IMPORT) if spec is None: raise SchemaUnavailable( @@ -118,6 +126,7 @@ def _package_root() -> Path: def _direct_url_payload(dist: importlib.metadata.Distribution) -> dict | None: + """Parse PEP 610 `direct_url.json` metadata when available.""" try: raw = dist.read_text("direct_url.json") except Exception: @@ -132,11 +141,13 @@ def _direct_url_payload(dist: importlib.metadata.Distribution) -> dict | None: def _normalize_repo(url: str) -> str: + """Normalize git URL for comparisons by trimming trailing slash and `.git` suffix.""" base = url.rstrip("/") return base[:-4] if base.endswith(".git") else base def _schema_info_from_install() -> SchemaInfo: + """Validate installed package provenance and return source/version metadata.""" dist = _distribution() package_root = _package_root() if str(package_root.parent) not in sys.path: @@ -146,6 +157,7 @@ def _schema_info_from_install() -> SchemaInfo: if not direct_url: return SchemaInfo(package_root=package_root, version=dist.version, source="installed") + # Light mode intentionally forbids local editable sources to keep behavior reproducible. source_url = direct_url.get("url") if not isinstance(source_url, str) or source_url.startswith("file://"): raise SchemaUnavailable(f"Light Mode does not support local {PACKAGE_DIST} sources.") @@ -159,6 +171,7 @@ def _schema_info_from_install() -> SchemaInfo: if not isinstance(vcs_info, dict): return SchemaInfo(package_root=package_root, version=dist.version, source="installed") + # If pip recorded a requested branch/tag, enforce the profile's fixed branch. requested = vcs_info.get("requested_revision") if requested and requested != DEFAULT_BRANCH: raise SchemaUnavailable( diff --git a/api/light_mode/tests/test_schema_source_profiles.py b/api/light_mode/tests/test_schema_source_profiles.py index a9cd107..6541500 100644 --- a/api/light_mode/tests/test_schema_source_profiles.py +++ b/api/light_mode/tests/test_schema_source_profiles.py @@ -11,6 +11,7 @@ def _reload_schema_source(monkeypatch, *, profile: str | None = None, package_hint: str | None = None): + """Reload schema-source module with controlled profile env vars.""" monkeypatch.delenv("SCHEMA_STUDIO_LIGHT_SCHEMA_PROFILE", raising=False) monkeypatch.delenv("SCHEMA_STUDIO_DEFAULT_PACKAGE", raising=False) @@ -26,6 +27,7 @@ def _reload_schema_source(monkeypatch, *, profile: str | None = None, package_hi def test_default_profile_is_nomad(monkeypatch): + """Without overrides, light mode should select NOMAD defaults.""" mod = _reload_schema_source(monkeypatch) assert mod.LIGHT_PROFILE_KEY == "nomad" @@ -35,6 +37,7 @@ def test_default_profile_is_nomad(monkeypatch): def test_explicit_bam_profile(monkeypatch): + """Explicit BAM profile should switch branch/package defaults.""" mod = _reload_schema_source(monkeypatch, profile="bam") assert mod.LIGHT_PROFILE_KEY == "bam" @@ -45,6 +48,7 @@ def test_explicit_bam_profile(monkeypatch): def test_package_hint_selects_bam_profile(monkeypatch): + """A BAM package hint should auto-select BAM profile when key is unset.""" mod = _reload_schema_source(monkeypatch, package_hint="bam_masterdata.datamodel.vocabulary_types") assert mod.LIGHT_PROFILE_KEY == "bam" diff --git a/api/repo_utils.py b/api/repo_utils.py index dcf03c3..566c88a 100644 --- a/api/repo_utils.py +++ b/api/repo_utils.py @@ -47,6 +47,7 @@ def parse_base_packages(raw: str) -> list[str]: def bases_by_repo(base_packages: list[str]) -> dict[str, list[str]]: + """Group base namespaces by owning repository path/URL.""" mapping: dict[str, list[str]] = {} for base in base_packages: repo = repo_for_base_namespace(base) diff --git a/api/settings.py b/api/settings.py index dfa43e0..51d5445 100644 --- a/api/settings.py +++ b/api/settings.py @@ -95,6 +95,7 @@ def _parse_repo_map(raw: str | None) -> list[tuple[str, str]]: repo = repo.strip() if prefix and repo: pairs.append((prefix, repo)) + # Longest-prefix first so specific namespaces override broader ones. pairs.sort(key=lambda p: len(p[0]), reverse=True) return pairs diff --git a/extractor/graph_builder.py b/extractor/graph_builder.py index d798f99..95980d7 100644 --- a/extractor/graph_builder.py +++ b/extractor/graph_builder.py @@ -6,6 +6,7 @@ @dataclass class Node: + """Serializable graph node representation for sections and quantities.""" id: str kind: str # "section" | "quantity" label: str @@ -20,6 +21,7 @@ class Node: @dataclass class Edge: + """Serializable directed relation between two graph nodes.""" source: str target: str type: str # "hasQuantity" | "hasSubSection" | "inherits" @@ -66,6 +68,7 @@ def _doc_from(obj: Any) -> Optional[str]: # -------- main API -------- def list_sections(package: str) -> List[str]: + """List discoverable section-like classes defined in the given module.""" mod = importlib.import_module(package) base_ns = _root_namespace(package) return sorted( @@ -219,14 +222,17 @@ def add_section(sec_obj: Any, depth: int = 0): # -------- introspection helpers -------- def _is_section(obj: Any) -> bool: + """Return True for classes that behave like NOMAD sections or BAM entities.""" return inspect.isclass(obj) and (_is_nomad_section(obj) or _is_bam_entity_class(obj)) def _is_nomad_section(obj: Any) -> bool: + """Detect NOMAD section classes via common metainfo attributes.""" return hasattr(obj, "m_def") or hasattr(obj, "quantities") or hasattr(obj, "sub_sections") def _is_bam_entity_class(obj: Any) -> bool: + """Detect BAM datamodel classes derived from ObjectType or VocabularyType.""" for base in getattr(obj, "__mro__", []): mod = getattr(base, "__module__", "") if mod != "bam_masterdata.metadata.entities": @@ -237,6 +243,7 @@ def _is_bam_entity_class(obj: Any) -> bool: def _is_bam_object_type_class(obj: Any) -> bool: + """Detect BAM object type classes derived from `ObjectType`.""" for base in getattr(obj, "__mro__", []): mod = getattr(base, "__module__", "") if mod != "bam_masterdata.metadata.entities": @@ -247,6 +254,7 @@ def _is_bam_object_type_class(obj: Any) -> bool: def _is_bam_vocabulary_type_class(obj: Any) -> bool: + """Detect BAM vocabulary classes derived from `VocabularyType`.""" for base in getattr(obj, "__mro__", []): mod = getattr(base, "__module__", "") if mod != "bam_masterdata.metadata.entities": @@ -257,14 +265,17 @@ def _is_bam_vocabulary_type_class(obj: Any) -> bool: def _is_bam_property_assignment(obj: Any) -> bool: + """Identify BAM property assignment descriptor instances.""" return obj.__class__.__name__ == "PropertyTypeAssignment" and obj.__class__.__module__ == "bam_masterdata.metadata.definitions" def _is_bam_vocabulary_term(obj: Any) -> bool: + """Identify BAM controlled vocabulary term descriptor instances.""" return obj.__class__.__name__ == "VocabularyTerm" and obj.__class__.__module__ == "bam_masterdata.metadata.definitions" def _items_from_mapping_or_list(x) -> Iterable[Tuple[str, Any]]: + """Normalize dict/list descriptor containers to `(name, object)` pairs.""" if x is None: return [] if isinstance(x, dict): @@ -280,6 +291,7 @@ def _items_from_mapping_or_list(x) -> Iterable[Tuple[str, Any]]: def _cardinality_from(obj) -> Optional[str]: + """Best-effort conversion of source cardinality fields to UML-style ranges.""" if hasattr(obj, "repeats"): try: return "0..*" if bool(getattr(obj, "repeats")) else "0..1" @@ -305,6 +317,7 @@ def _ref_target_name(dtype_obj) -> Optional[str]: """Best-effort human-friendly target for Reference-like dtypes.""" def _name_from_target(target: Any) -> Optional[str]: + """Resolve a readable section name from a referenced target object.""" cls = _resolve_section_class(target) if cls is None: cls = target if inspect.isclass(target) else getattr(target, "__class__", None) @@ -337,6 +350,7 @@ def _name_from_target(target: Any) -> Optional[str]: def _enumish_value(value: Any) -> str: + """Convert enum-like values into a stable scalar string.""" enum_value = getattr(value, "value", None) if isinstance(enum_value, str) and enum_value: return enum_value @@ -347,6 +361,7 @@ def _enumish_value(value: Any) -> str: def _dtype_from(q) -> Optional[str]: + """Extract a display dtype from NOMAD/BAM quantity-like objects.""" if _is_bam_vocabulary_term(q): # Vocabulary terms are similar to enums, but we want to preserve the link to the vocabulary return "VOCAB_TERM" @@ -378,6 +393,7 @@ def _dtype_from(q) -> Optional[str]: def _shape_from(q) -> Optional[str]: + """Extract quantity shape metadata when available.""" if hasattr(q, "shape"): try: return str(getattr(q, "shape")) @@ -387,9 +403,11 @@ def _shape_from(q) -> Optional[str]: def _get_bam_quantities(sec_obj: Any) -> Iterable[Tuple[str, Any]]: + """Collect BAM properties/terms from class dictionaries across inheritance.""" collected: Dict[str, Any] = {} if _is_bam_object_type_class(sec_obj): + # Traverse MRO from base to derived so child classes can override inherited fields. for base in reversed(getattr(sec_obj, "__mro__", [])): for attr_name, attr_value in getattr(base, "__dict__", {}).items(): if _is_bam_property_assignment(attr_value): @@ -407,6 +425,7 @@ def _get_bam_quantities(sec_obj: Any) -> Iterable[Tuple[str, Any]]: def _get_quantities(sec_obj) -> Iterable[Tuple[str, Any]]: + """Return normalized quantity pairs from NOMAD or BAM section definitions.""" qmap = getattr(sec_obj, "quantities", None) if qmap: return _items_from_mapping_or_list(qmap) @@ -426,6 +445,7 @@ def _get_quantities(sec_obj) -> Iterable[Tuple[str, Any]]: def _get_subsections(sec_obj) -> Iterable[Tuple[str, Any]]: + """Return normalized subsection pairs from NOMAD-style section definitions.""" smap = getattr(sec_obj, "sub_sections", None) if smap: return _items_from_mapping_or_list(smap) @@ -481,6 +501,7 @@ def _resolve_section_class(target) -> Optional[type]: def _root_namespace(package: str) -> str: + """Derive a stable namespace prefix used for module filtering.""" parts = package.split(".") if len(parts) >= 3: return ".".join(parts[:3]) @@ -490,6 +511,7 @@ def _root_namespace(package: str) -> str: def _module_allowed(module: str, base_namespace: str, exclude_prefixes: Tuple[str, ...], allow_cross_module: bool) -> bool: + """Decide whether a module should be traversed during graph expansion.""" if any(module.startswith(p) for p in exclude_prefixes): return False if allow_cross_module: @@ -498,6 +520,7 @@ def _module_allowed(module: str, base_namespace: str, exclude_prefixes: Tuple[st def _module_in_namespace(obj, package_namespace: str) -> bool: + """Check whether the object's module belongs to the target namespace.""" mod = getattr(obj, "__module__", "") return mod.startswith(package_namespace) diff --git a/extractor/tests/test_graph_builder_bam.py b/extractor/tests/test_graph_builder_bam.py index 07dbfaf..ea89f2e 100644 --- a/extractor/tests/test_graph_builder_bam.py +++ b/extractor/tests/test_graph_builder_bam.py @@ -13,6 +13,7 @@ def _write_fake_bam_package(root: Path) -> None: + """Create a minimal BAM-like package tree used for extractor unit tests.""" files: dict[str, str] = { "bam_masterdata/__init__.py": "", "bam_masterdata/metadata/__init__.py": "", @@ -112,6 +113,7 @@ class DeviceStatus(VocabularyType): def test_build_graph_extracts_bam_object_types_and_controlled_vocab(monkeypatch, tmp_path: Path): + """BAM object type fields should become quantity nodes with typed cardinalities.""" _write_fake_bam_package(tmp_path) monkeypatch.syspath_prepend(str(tmp_path)) importlib.invalidate_caches() @@ -150,6 +152,7 @@ def test_build_graph_extracts_bam_object_types_and_controlled_vocab(monkeypatch, def test_build_graph_extracts_bam_vocabulary_terms(monkeypatch, tmp_path: Path): + """BAM vocabulary terms should be extracted as quantity-like term nodes.""" _write_fake_bam_package(tmp_path) monkeypatch.syspath_prepend(str(tmp_path)) importlib.invalidate_caches() diff --git a/extractor/tests/test_settings_repo_mapping.py b/extractor/tests/test_settings_repo_mapping.py index 58a12a6..01f2b38 100644 --- a/extractor/tests/test_settings_repo_mapping.py +++ b/extractor/tests/test_settings_repo_mapping.py @@ -11,6 +11,7 @@ def _reload_settings(monkeypatch, **env): + """Reload `api.settings` with a clean, test-specific environment.""" keys = { "SCHEMA_UML_REPO", "NOMAD_SIM_REPO", @@ -34,6 +35,7 @@ def _reload_settings(monkeypatch, **env): def test_bam_repo_is_selected_by_namespace(monkeypatch, tmp_path: Path): + """BAM namespaces should resolve to BAM repo when configured.""" default_repo = tmp_path / "default" bam_repo = tmp_path / "bam" @@ -49,6 +51,7 @@ def test_bam_repo_is_selected_by_namespace(monkeypatch, tmp_path: Path): def test_repo_map_overrides_default_namespace_mapping(monkeypatch, tmp_path: Path): + """Explicit namespace mappings should override built-in repo defaults.""" default_repo = tmp_path / "default" bam_repo = tmp_path / "bam" mapped_repo = tmp_path / "mapped" @@ -65,6 +68,7 @@ def test_repo_map_overrides_default_namespace_mapping(monkeypatch, tmp_path: Pat def test_default_package_for_bam_namespace(monkeypatch, tmp_path: Path): + """BAM base namespace should default to the object types module.""" default_repo = tmp_path / "default" mod = _reload_settings(