From f4b83503bf044c83b270bbec5405fdd599fe8fed Mon Sep 17 00:00:00 2001 From: Dragos Crintea Date: Wed, 15 Jul 2026 14:16:05 +0100 Subject: [PATCH 1/5] add semantido as a ossie vendor --- python/src/ossie/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/src/ossie/models.py b/python/src/ossie/models.py index b0db1cc0..1c8b6465 100644 --- a/python/src/ossie/models.py +++ b/python/src/ossie/models.py @@ -43,6 +43,7 @@ class OSIVendor(str, Enum): DBT = "DBT" DATABRICKS = "DATABRICKS" GOODDATA = "GOODDATA" + SEMANTIDO = "SEMANTIDO" class OSIAIContextObject(BaseModel): From bc65bdd28053cbc618f8d8499c05b570d40bc691 Mon Sep 17 00:00:00 2001 From: Dragos Crintea Date: Sat, 18 Jul 2026 23:03:52 +0100 Subject: [PATCH 2/5] semantido <-> ossie converter --- converters/semantido/README.md | 86 +++++++ converters/semantido/pyproject.toml | 53 ++++ .../semantido/src/ossie_semantido/__init__.py | 32 +++ .../semantido/src/ossie_semantido/cli.py | 87 +++++++ .../src/ossie_semantido/converter_issues.py | 51 ++++ .../semantido/src/ossie_semantido/loaders.py | 46 ++++ .../src/ossie_semantido/osi_to_semantido.py | 153 ++++++++++++ .../src/ossie_semantido/semantido_to_osi.py | 190 +++++++++++++++ converters/semantido/tests/__init__.py | 0 .../semantido/tests/fixtures/__init__.py | 0 .../tests/fixtures/emir_reporting/__init__.py | 0 .../emir_reporting/models/__init__.py | 0 .../emir_reporting/models/emir_reporting.py | 226 ++++++++++++++++++ converters/semantido/tests/helpers.py | 30 +++ .../semantido/tests/test_osi_to_semantido.py | 37 +++ .../semantido/tests/test_semantido_to_osi.py | 76 ++++++ 16 files changed, 1067 insertions(+) create mode 100644 converters/semantido/README.md create mode 100644 converters/semantido/pyproject.toml create mode 100644 converters/semantido/src/ossie_semantido/__init__.py create mode 100644 converters/semantido/src/ossie_semantido/cli.py create mode 100644 converters/semantido/src/ossie_semantido/converter_issues.py create mode 100644 converters/semantido/src/ossie_semantido/loaders.py create mode 100644 converters/semantido/src/ossie_semantido/osi_to_semantido.py create mode 100644 converters/semantido/src/ossie_semantido/semantido_to_osi.py create mode 100644 converters/semantido/tests/__init__.py create mode 100644 converters/semantido/tests/fixtures/__init__.py create mode 100644 converters/semantido/tests/fixtures/emir_reporting/__init__.py create mode 100644 converters/semantido/tests/fixtures/emir_reporting/models/__init__.py create mode 100644 converters/semantido/tests/fixtures/emir_reporting/models/emir_reporting.py create mode 100644 converters/semantido/tests/helpers.py create mode 100644 converters/semantido/tests/test_osi_to_semantido.py create mode 100644 converters/semantido/tests/test_semantido_to_osi.py diff --git a/converters/semantido/README.md b/converters/semantido/README.md new file mode 100644 index 00000000..fb11b97b --- /dev/null +++ b/converters/semantido/README.md @@ -0,0 +1,86 @@ + + +# semantido ⇄ Apache Ossie converter + +Converts between [semantido](https://github.com/hikarilabs/semantido) — a +code-native semantic layer authored as decorators on SQLAlchemy models and +Apache Ossie semantic model documents. + +Unlike file-to-file converters, semantido's source of truth is live Python: +the forward direction imports a module of decorated models, syncs the +semantic layer, and emits Ossie YAML through the typed `apache-ossie` +objects. The reverse direction is code generation: it produces a Python +module of `@semantic_table`-decorated models from an Ossie document. + +## Usage + +```bash +# decorated SQLAlchemy models -> Ossie YAML +ossie-semantido semantido-to-osi \ + -m models.[SQLAlchemy model] -p ./my_project \ + -n [model name] -o [model].osi.yaml + +# Ossie YAML -> generated semantido model code +ossie-semantido osi-to-semantido \ + -i [model].osi.yaml -o generated_model_[model name].py +``` + +## Mapping + +| semantido | Ossie | Notes | +|-------------------------------------------|-----------------------------------|------------------------------------------------------| +| decorated table | `datasets[]` | `source` from schema-qualified table name | +| column | `fields[]` | expression emitted as `ANSI_SQL` column reference | +| `description` / `_description` | `description` | | +| `business_context`, `application_context` | dataset `ai_context.instructions` | | +| `synonyms` / `_synonyms` | `ai_context.synonyms` | | +| `_sample_values` | field `ai_context.examples` | | +| `time_dimension=` + `_time_grain` | field `dimension.is_time` | grain preserved in extensions | +| FK relationships | `relationships[]` | join condition parsed to `from_columns`/`to_columns` | +| `sql_filters` | dataset `custom_extensions` | no Ossie core field (see below) | +| `_privacy_level` | field `custom_extensions` | no Ossie core field (see below) | + +### Metadata carried in `custom_extensions` + +semantido captures runtime-governance metadata that has no core-spec home +in Ossie today: per-field privacy classification, default SQL filters / +row-level security fragments, and time-dimension grain. These are preserved +losslessly under `vendor_name: SEMANTIDO` with `data` as a serialized JSON +string, so no information is lost in interchange — but tools without +semantido awareness will not interpret them. + +### Known lossy conversions + +Conversions report structured `ConverterIssue`s (mirroring `ossie_dbt`): +Ossie metrics and non-ANSI dialect expressions have no semantido +equivalent and are dropped with a recorded issue; generated code lists +them in a TODO block for review. + +## Tests + +The test fixture is an EMIR (European Market Infrastructure Regulation) +trade-reporting model — a regulated-industry schema whose semantics +(role-bridge fan-out, signed vs. unsigned amount conventions, +notional/valuation ambiguity) demonstrate why interchange must carry +business meaning, not just structure. + +```bash +uv run pytest tests/ +``` diff --git a/converters/semantido/pyproject.toml b/converters/semantido/pyproject.toml new file mode 100644 index 00000000..89f5d54e --- /dev/null +++ b/converters/semantido/pyproject.toml @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "apache-ossie-semantido" +version = "0.1.0.dev0" +description = "semantido (SQLAlchemy code-native semantic layer) <> Apache Ossie converter" +requires-python = ">=3.11" +classifiers = [ + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", +] +dependencies = [ + "apache-ossie>=0.2.0.dev0", + "semantido>=0.4.0", + "SQLAlchemy>=2.0", + "PyYAML>=6.0", +] + +[project.license] +text = "Apache-2.0" + +[project.scripts] +ossie-semantido = "ossie_semantido.cli:main" + +[tool.hatch.build.targets.wheel] +packages = ["src/ossie_semantido"] + +[tool.uv] +dev-dependencies = [ + "pytest>=8.0", +] + +[tool.uv.sources] +apache-ossie = { path = "../../python", editable = true } diff --git a/converters/semantido/src/ossie_semantido/__init__.py b/converters/semantido/src/ossie_semantido/__init__.py new file mode 100644 index 00000000..8a4f9377 --- /dev/null +++ b/converters/semantido/src/ossie_semantido/__init__.py @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""semantido <> Apache Ossie converter.""" + +from ossie_semantido.converter_issues import ConverterIssue, ConverterIssueType, ConverterResult +from ossie_semantido.loaders import load_from_module +from ossie_semantido.osi_to_semantido import osi_to_semantido_source +from ossie_semantido.semantido_to_osi import semantic_layer_to_osi + +__all__ = [ + "ConverterIssue", + "ConverterIssueType", + "ConverterResult", + "load_from_module", + "osi_to_semantido_source", + "semantic_layer_to_osi", +] diff --git a/converters/semantido/src/ossie_semantido/cli.py b/converters/semantido/src/ossie_semantido/cli.py new file mode 100644 index 00000000..7e26df68 --- /dev/null +++ b/converters/semantido/src/ossie_semantido/cli.py @@ -0,0 +1,87 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Command-line interface for the semantido <> Apache Ossie converter.""" + +import argparse +import sys + +import yaml +from ossie import OSIDocument + +from ossie_semantido.loaders import load_from_module +from ossie_semantido.osi_to_semantido import osi_to_semantido_source +from ossie_semantido.semantido_to_osi import semantic_layer_to_osi + + +def _report_issues(issues) -> None: + for issue in issues: + print(f"warning: {issue.issue_type.value}: {issue.element_name}", file=sys.stderr) + + +def _cmd_semantido_to_osi(args: argparse.Namespace) -> None: + layer = load_from_module(args.module, sys_path=args.path) + result = semantic_layer_to_osi(layer, model_name=args.name) + _report_issues(result.issues) + with open(args.output, "w", encoding="utf-8") as handle: + handle.write(result.output.to_osi_yaml()) + + +def _cmd_osi_to_semantido(args: argparse.Namespace) -> None: + with open(args.input, "r", encoding="utf-8") as handle: + document = OSIDocument.model_validate(yaml.safe_load(handle)) + result = osi_to_semantido_source(document) + _report_issues(result.issues) + with open(args.output, "w", encoding="utf-8") as handle: + handle.write(result.output) + + +def main() -> None: + parser = argparse.ArgumentParser( + prog="ossie-semantido", + description="Convert between semantido semantic layers and Apache Ossie documents", + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + to_osi = subparsers.add_parser( + "semantido-to-osi", help="Convert decorated SQLAlchemy models → Ossie YAML" + ) + to_osi.add_argument("-m", "--module", required=True, metavar="MODULE", + help="Dotted module path of the semantido models, e.g. models.emir_reporting") + to_osi.add_argument("-p", "--path", default=None, metavar="DIR", + help="Directory prepended to sys.path before import") + to_osi.add_argument("-n", "--name", required=True, metavar="NAME", + help="Name for the Ossie semantic_model") + to_osi.add_argument("-o", "--output", required=True, metavar="FILE", + help="Path for output Ossie YAML") + to_osi.set_defaults(func=_cmd_semantido_to_osi) + + from_osi = subparsers.add_parser( + "osi-to-semantido", help="Convert Ossie YAML → generated semantido model code" + ) + from_osi.add_argument("-i", "--input", required=True, metavar="FILE", + help="Path to Ossie YAML") + from_osi.add_argument("-o", "--output", required=True, metavar="FILE", + help="Path for generated Python module") + from_osi.set_defaults(func=_cmd_osi_to_semantido) + + args = parser.parse_args() + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/converters/semantido/src/ossie_semantido/converter_issues.py b/converters/semantido/src/ossie_semantido/converter_issues.py new file mode 100644 index 00000000..e35fbfb2 --- /dev/null +++ b/converters/semantido/src/ossie_semantido/converter_issues.py @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Issue reporting for lossy conversions, mirroring the ossie_dbt pattern.""" + +from dataclasses import dataclass +from enum import Enum +from typing import Generic, List, TypeVar + + +class ConverterIssueType(Enum): + """Identifies the kind of information loss that occurred during conversion.""" + + RELATIONSHIP_JOIN_UNPARSED = "RELATIONSHIP_JOIN_UNPARSED" + METRIC_DROPPED = "METRIC_DROPPED" + NON_ANSI_EXPRESSION_DROPPED = "NON_ANSI_EXPRESSION_DROPPED" + COMPOSITE_PRIMARY_KEY_TRUNCATED = "COMPOSITE_PRIMARY_KEY_TRUNCATED" + UNMAPPED_DATA_TYPE = "UNMAPPED_DATA_TYPE" + + +@dataclass(frozen=True) +class ConverterIssue: + """Records a single instance of information loss during conversion.""" + + issue_type: ConverterIssueType + element_name: str + + +T = TypeVar("T") + + +@dataclass(frozen=True) +class ConverterResult(Generic[T]): + """Return value of a converter's convert() method, pairing the output with any conversion issues.""" + + output: T + issues: List[ConverterIssue] diff --git a/converters/semantido/src/ossie_semantido/loaders.py b/converters/semantido/src/ossie_semantido/loaders.py new file mode 100644 index 00000000..eba7c85a --- /dev/null +++ b/converters/semantido/src/ossie_semantido/loaders.py @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Entry points for loading a semantido SemanticLayer. + +semantido's source of truth is live Python (decorated SQLAlchemy models), +so the primary loader imports a module and syncs the layer. A file-based +path is not provided in this converter: semantido's JSON export is a +rendering of the same layer, and consumers who have the JSON already have +a semantic model they can transform directly. +""" + +import importlib +import sys +from pathlib import Path + +from semantido import SemanticDeclarativeBase +from semantido.generators.semantic_layer import SemanticLayer + + +def load_from_module(module_path: str, sys_path: str | None = None) -> SemanticLayer: + """Import a module of decorated models and return the synced layer. + + Args: + module_path: Dotted module path, e.g. ``models.emir_reporting``. + sys_path: Optional directory prepended to ``sys.path`` first, so + model packages can be loaded without installation. + """ + if sys_path: + sys.path.insert(0, str(Path(sys_path).resolve())) + importlib.import_module(module_path) + return SemanticDeclarativeBase.sync_semantic_layer() diff --git a/converters/semantido/src/ossie_semantido/osi_to_semantido.py b/converters/semantido/src/ossie_semantido/osi_to_semantido.py new file mode 100644 index 00000000..91a4aa2c --- /dev/null +++ b/converters/semantido/src/ossie_semantido/osi_to_semantido.py @@ -0,0 +1,153 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Apache Ossie document -> generated semantido model code. + +This direction is code generation: it emits a Python module of +``@semantic_table``-decorated SQLAlchemy models. Constructs with no +semantido equivalent (metrics, non-ANSI dialect expressions) are dropped +with a recorded ConverterIssue and listed in a TODO comment block at the +top of the generated file. +""" + +import json +from typing import List + +from ossie import OSIDataset, OSIDocument, OSIField + +from ossie_semantido.converter_issues import ConverterIssue, ConverterIssueType, ConverterResult + +VENDOR_NAME = "SEMANTIDO" + +_TYPE_MAP = { + "VARCHAR": "String", + "CHAR": "String", + "TEXT": "String", + "INTEGER": "Integer", + "BIGINT": "Integer", + "SMALLINT": "Integer", + "NUMERIC": "Numeric", + "DECIMAL": "Numeric", + "FLOAT": "Float", + "DATE": "Date", + "DATETIME": "DateTime", + "TIMESTAMP": "DateTime", + "BOOLEAN": "Boolean", +} + + +def _vendor_payload(entity) -> dict: + for ext in entity.custom_extensions or []: + if ext.vendor_name == VENDOR_NAME: + try: + return json.loads(ext.data) + except (TypeError, ValueError): + return {} + return {} + + +def _sqlalchemy_type(field: OSIField, issues: List[ConverterIssue]) -> str: + data_type = _vendor_payload(field).get("data_type", "") + base = data_type.split("(")[0].strip().upper() + if base in _TYPE_MAP: + return _TYPE_MAP[base] + if base: + issues.append( + ConverterIssue(ConverterIssueType.UNMAPPED_DATA_TYPE, f"{field.name}:{base}") + ) + return "String" + + +def _ai(entity): + ctx = entity.ai_context + if ctx is None or isinstance(ctx, str): + return None + return ctx + + +def _class_name(dataset_name: str) -> str: + return "".join(part.capitalize() for part in dataset_name.split("_")) + + +def _emit_dataset(dataset: OSIDataset, issues: List[ConverterIssue]) -> str: + payload = _vendor_payload(dataset) + ctx = _ai(dataset) + + decorator_kwargs = [] + if dataset.description: + decorator_kwargs.append(f" description={dataset.description!r},") + if ctx and ctx.instructions: + decorator_kwargs.append(f" business_context={ctx.instructions!r},") + if ctx and ctx.synonyms: + decorator_kwargs.append(f" synonyms={list(ctx.synonyms)!r},") + if payload.get("sql_filters"): + decorator_kwargs.append(f" sql_filters={payload['sql_filters']!r},") + if payload.get("time_dimension"): + decorator_kwargs.append(f" time_dimension={payload['time_dimension']!r},") + + lines = ["@semantic_table("] + decorator_kwargs + [")"] + lines.append(f"class {_class_name(dataset.name)}(SemanticDeclarativeBase):") + lines.append(f' __tablename__ = "{dataset.name}"') + lines.append("") + + primary_keys = set(dataset.primary_key or []) + for field in dataset.fields or []: + field_payload = _vendor_payload(field) + field_ctx = _ai(field) + sa_type = _sqlalchemy_type(field, issues) + pk = ", primary_key=True" if field.name in primary_keys else "" + lines.append(f" {field.name} = Column({sa_type}{pk})") + if field.description: + lines.append(f" {field.name}_description = {field.description!r}") + if field_payload.get("privacy_level"): + level = field_payload["privacy_level"].upper() + lines.append(f" {field.name}_privacy_level = PrivacyLevel.{level}") + if field_ctx and field_ctx.examples: + lines.append(f" {field.name}_sample_values = {[str(e) for e in field_ctx.examples]!r}") + if field_payload.get("time_grain"): + grain = field_payload["time_grain"].upper() + lines.append(f" {field.name}_time_grain = TimeGrain.{grain}") + return "\n".join(lines) + + +def osi_to_semantido_source(document: OSIDocument) -> ConverterResult[str]: + """Generate semantido model source code from an Ossie document.""" + issues: List[ConverterIssue] = [] + blocks = [] + + for model in document.semantic_model: + for metric in model.metrics or []: + issues.append(ConverterIssue(ConverterIssueType.METRIC_DROPPED, metric.name)) + for dataset in model.datasets: + blocks.append(_emit_dataset(dataset, issues)) + + todo_block = "" + if issues: + todo_lines = "\n".join(f"# {i.issue_type.value}: {i.element_name}" for i in issues) + todo_block = ( + "# TODO(ossie-semantido): the following Ossie constructs have no\n" + "# semantido equivalent yet and were not converted:\n" + todo_lines + "\n\n" + ) + + header = ( + '"""semantido models generated by ossie-semantido. Review before use."""\n\n' + "from sqlalchemy import Boolean, Column, Date, DateTime, Float, Integer, Numeric, String\n\n" + "from semantido import SemanticDeclarativeBase, semantic_table\n" + "from semantido.generators.semantic_layer import PrivacyLevel, TimeGrain\n\n\n" + ) + source = header + todo_block + "\n\n\n".join(blocks) + "\n" + return ConverterResult(output=source, issues=issues) diff --git a/converters/semantido/src/ossie_semantido/semantido_to_osi.py b/converters/semantido/src/ossie_semantido/semantido_to_osi.py new file mode 100644 index 00000000..74743b85 --- /dev/null +++ b/converters/semantido/src/ossie_semantido/semantido_to_osi.py @@ -0,0 +1,190 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""semantido SemanticLayer -> Apache Ossie document conversion. + +Builds the Ossie model through the typed ``apache-ossie`` objects so the +output is schema-conformant by construction. Governance metadata that has +no Ossie core field (privacy levels, SQL filters, time grain, the primary +time axis) is preserved losslessly in ``custom_extensions`` under the +``SEMANTIDO`` vendor name, with ``data`` as a serialized JSON string as +the specification requires. +""" + +import json +import re +from typing import List, Optional + +from ossie import ( + OSIAIContextObject, + OSICustomExtension, + OSIDataset, + OSIDialect, + OSIDialectExpression, + OSIDimension, + OSIDocument, + OSIExpression, + OSIField, + OSIRelationship, + OSISemanticModel, +) +from semantido.generators.semantic_layer import Column, Relationship, SemanticLayer, Table + +from ossie_semantido.converter_issues import ConverterIssue, ConverterIssueType, ConverterResult + +VENDOR_NAME = "SEMANTIDO" + +_JOIN_RE = re.compile( + r"^\s*(\w+)\.(\w+)\s*=\s*(\w+)\.(\w+)\s*$" +) + + +def _extension(payload: dict) -> OSICustomExtension: + """Wrap semantido-specific metadata as a spec-conformant custom extension.""" + return OSICustomExtension( + vendor_name=VENDOR_NAME, + data=json.dumps(payload, sort_keys=True, default=str), + ) + + +def _ansi(expression: str) -> OSIExpression: + return OSIExpression( + dialects=[OSIDialectExpression(dialect=OSIDialect.ANSI_SQL, expression=expression)] + ) + + +def _column_to_field(column: Column) -> OSIField: + ai_kwargs = {} + if column.synonyms: + ai_kwargs["synonyms"] = tuple(column.synonyms) + if column.sample_values: + ai_kwargs["examples"] = tuple(str(v) for v in column.sample_values) + if column.application_rules: + ai_kwargs["instructions"] = " ".join(column.application_rules) + + extension_payload = {} + if column.privacy_level is not None: + extension_payload["privacy_level"] = column.privacy_level.value + if column.time_grain is not None: + extension_payload["time_grain"] = column.time_grain.value + if column.data_type: + extension_payload["data_type"] = column.data_type + if column.references: + extension_payload["references"] = column.references + + return OSIField( + name=column.name, + expression=_ansi(column.name), + dimension=OSIDimension(is_time=True) if column.is_time_dimension else None, + description=column.description or None, + ai_context=OSIAIContextObject(**ai_kwargs) if ai_kwargs else None, + custom_extensions=[_extension(extension_payload)] if extension_payload else None, + ) + + +def _table_to_dataset(table: Table) -> OSIDataset: + instructions_parts = [p for p in (table.business_context, table.application_context) if p] + + ai_kwargs = {} + if instructions_parts: + ai_kwargs["instructions"] = " ".join(instructions_parts) + if table.synonyms: + ai_kwargs["synonyms"] = tuple(table.synonyms) + + extension_payload = {} + if table.sql_filters: + extension_payload["sql_filters"] = table.sql_filters + if table.time_dimension: + extension_payload["time_dimension"] = table.time_dimension + + return OSIDataset( + name=table.name, + source=f"{table.schema}.{table.name}" if table.schema else table.name, + primary_key=table.primary_key if table.primary_key else None, + description=table.description or None, + ai_context=OSIAIContextObject(**ai_kwargs) if ai_kwargs else None, + fields=[_column_to_field(c) for c in table.columns] or None, + custom_extensions=[_extension(extension_payload)] if extension_payload else None, + ) + + +def _relationship_to_osi( + rel: Relationship, issues: List[ConverterIssue] +) -> Optional[OSIRelationship]: + match = _JOIN_RE.match(rel.join_condition or "") + if not match: + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.RELATIONSHIP_JOIN_UNPARSED, + element_name=f"{rel.from_table}->{rel.to_table}", + ) + ) + return None + + left_table, left_col, right_table, right_col = match.groups() + # Normalize so from/to match the declared direction regardless of + # which side of the equality each table appeared on. + if left_table == rel.from_table: + from_columns, to_columns = [left_col], [right_col] + else: + from_columns, to_columns = [right_col], [left_col] + + ai_kwargs = {} + if rel.description: + ai_kwargs["instructions"] = rel.description + + return OSIRelationship.model_validate( + { + "name": f"{rel.from_table}_to_{rel.to_table}", + "from": rel.from_table, + "to": rel.to_table, + "from_columns": from_columns, + "to_columns": to_columns, + "ai_context": ai_kwargs or None, + "custom_extensions": [ + _extension({"relationship_type": rel.relationship_type.value}).model_dump() + ], + } + ) + + +def semantic_layer_to_osi( + layer: SemanticLayer, model_name: str, description: Optional[str] = None +) -> ConverterResult[OSIDocument]: + """Convert a synced semantido SemanticLayer into an Ossie document.""" + issues: List[ConverterIssue] = [] + + datasets = [_table_to_dataset(t) for t in layer.tables.values()] + + relationships = [] + for rel in layer.relationships: + converted = _relationship_to_osi(rel, issues) + if converted is not None: + relationships.append(converted) + + model_extensions = None + if layer.application_glossary: + model_extensions = [_extension({"application_glossary": layer.application_glossary})] + + model = OSISemanticModel( + name=model_name, + description=description, + datasets=datasets, + relationships=relationships or None, + custom_extensions=model_extensions, + ) + return ConverterResult(output=OSIDocument(semantic_model=[model]), issues=issues) diff --git a/converters/semantido/tests/__init__.py b/converters/semantido/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/converters/semantido/tests/fixtures/__init__.py b/converters/semantido/tests/fixtures/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/converters/semantido/tests/fixtures/emir_reporting/__init__.py b/converters/semantido/tests/fixtures/emir_reporting/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/converters/semantido/tests/fixtures/emir_reporting/models/__init__.py b/converters/semantido/tests/fixtures/emir_reporting/models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/converters/semantido/tests/fixtures/emir_reporting/models/emir_reporting.py b/converters/semantido/tests/fixtures/emir_reporting/models/emir_reporting.py new file mode 100644 index 00000000..2485e3af --- /dev/null +++ b/converters/semantido/tests/fixtures/emir_reporting/models/emir_reporting.py @@ -0,0 +1,226 @@ +"""EMIR trade-reporting semantic model (regulated-industry fixture). + +Simplified but structurally honest EMIR REFIT reporting schema. It +deliberately encodes three semantics-dependent failure modes that plain +DDL cannot express: bridge fan-out via the counterparty role table, sign +conventions (signed valuations vs. unsigned posted/received collateral), +and amount ambiguity (notional vs. valuation vs. collateral). +""" + +from sqlalchemy import Column, Date, DateTime, ForeignKey, Integer, Numeric, String +from sqlalchemy.orm import relationship + +from semantido import SemanticDeclarativeBase, semantic_table +from semantido.generators.semantic_layer import PrivacyLevel, TimeGrain + + +@semantic_table( + description="Legal entity master for derivative counterparties, one row per LEI.", + business_context=( + "EMIR counterparty classification drives clearing and reporting " + "obligations (FC / NFC+ / NFC- under Art. 2 and Art. 10)." + ), + synonyms=["legal entities", "counterparties"], +) +class Counterparty(SemanticDeclarativeBase): + __tablename__ = "counterparty" + + lei = Column(String(20), primary_key=True) + lei_description = ( + "Legal Entity Identifier (ISO 17442). The sole join key for " + "counterparty identity everywhere in this schema." + ) + lei_sample_values = ["529900T8BM49AURSDO55"] + + legal_name = Column(String(200), nullable=False) + legal_name_description = "Registered legal name as per GLEIF record." + legal_name_privacy_level = PrivacyLevel.CONFIDENTIAL + + classification = Column(String(4), nullable=False) + classification_description = ( + "EMIR counterparty classification: 'FC' (financial counterparty, Art. 2(8)), " + "'NFC+' (non-financial above clearing threshold, Art. 10), 'NFC-' (below). " + "Filter on this before any obligation question." + ) + classification_sample_values = ["FC", "NFC+"] + + country = Column(String(2), nullable=False) + country_description = "ISO 3166-1 alpha-2 country of incorporation." + + roles = relationship("CounterpartyTradeRole", back_populates="counterparty") + roles_relationship_description = "Roles this entity plays on trades." + + +@semantic_table( + description=( + "Reportable OTC derivative trades, one row per UTI. Immutable birth " + "record of the trade; evolving economics live in trade_state." + ), + business_context=( + "AMBIGUITY GUARD: notional_amount is contract size, NOT current value " + "and NOT exposure. 'How big' -> notional; 'worth today' -> " + "trade_state.valuation_amount; 'at risk' -> collateral_report." + ), + synonyms=["trades", "derivatives", "positions"], + time_dimension="execution_timestamp", +) +class Trade(SemanticDeclarativeBase): + __tablename__ = "trade" + + uti = Column(String(52), primary_key=True) + uti_description = "Unique Trade Identifier (Art. 9 REFIT). Canonical trade key." + + asset_class = Column(String(4), nullable=False) + asset_class_description = ( + "EMIR asset class: 'IR' rates, 'CR' credit, 'EQ' equity, 'FX' FX, 'CO' commodity." + ) + asset_class_sample_values = ["IR", "FX"] + + notional_amount = Column(Numeric(20, 2), nullable=False) + notional_amount_description = ( + "Trade notional in notional_currency. Contract size, not value. Never " + "sum across currencies without conversion." + ) + notional_amount_privacy_level = PrivacyLevel.RESTRICTED + + notional_currency = Column(String(3), nullable=False) + notional_currency_description = "ISO 4217 currency of notional_amount." + notional_currency_sample_values = ["EUR", "USD"] + + execution_timestamp = Column(DateTime, nullable=False) + execution_timestamp_description = "UTC execution time (Art. 9 REFIT). Primary time axis." + execution_timestamp_time_grain = TimeGrain.SECOND + + states = relationship("TradeState", back_populates="trade") + states_relationship_description = "Daily state reports for this trade." + roles = relationship("CounterpartyTradeRole", back_populates="trade") + roles_relationship_description = "Counterparty roles attached to this trade." + + +@semantic_table( + description=( + "Daily trade-state reports (EMIR REFIT is state-based), one row per " + "UTI per reporting_date. The latest state per UTI is the current view." + ), + business_context=( + "SIGN CONVENTION: valuation_amount is signed from the REPORTING " + "counterparty's perspective. Aggregating across counterparties without " + "flipping signs by role is meaningless." + ), + sql_filters=[ + "reporting_date = (SELECT MAX(ts2.reporting_date) FROM trade_state ts2 " + "WHERE ts2.uti = trade_state.uti)" + ], + time_dimension="reporting_date", +) +class TradeState(SemanticDeclarativeBase): + __tablename__ = "trade_state" + + id = Column(Integer, primary_key=True) + uti = Column(String(52), ForeignKey("trade.uti"), nullable=False) + reporting_date = Column(Date, nullable=False) + reporting_date_description = "State date; one state per trade per day." + reporting_date_time_grain = TimeGrain.DAY + + valuation_amount = Column(Numeric(20, 2)) + valuation_amount_description = ( + "Mark-to-market value (Art. 11(2)), signed from the reporting " + "counterparty's perspective. Value, not size: do not confuse with notional." + ) + valuation_amount_privacy_level = PrivacyLevel.RESTRICTED + + contract_status = Column(String(4), nullable=False) + contract_status_description = ( + "'OUTS' outstanding, 'TERM' terminated, 'MATU' matured, 'ERRO' error. " + "Exclude non-OUTS from exposure aggregates unless asked otherwise." + ) + contract_status_sample_values = ["OUTS", "TERM"] + + trade = relationship("Trade", back_populates="states") + trade_relationship_description = "The trade this state report belongs to." + + +@semantic_table( + description=( + "Role bridge between trades and counterparties. A trade has AT LEAST " + "two rows here (reporting + other counterparty), often more." + ), + business_context=( + "FAN-OUT GUARD: joining trade -> this table -> counterparty multiplies " + "trade rows by role count; any SUM over trade amounts after this join " + "double-counts. Filter to a single role (usually 'RPTG') before " + "aggregating trade-level amounts." + ), +) +class CounterpartyTradeRole(SemanticDeclarativeBase): + __tablename__ = "counterparty_trade_role" + + id = Column(Integer, primary_key=True) + uti = Column(String(52), ForeignKey("trade.uti"), nullable=False) + lei = Column(String(20), ForeignKey("counterparty.lei"), nullable=False) + + role = Column(String(4), nullable=False) + role_description = ( + "'RPTG' reporting counterparty, 'OTHR' other counterparty, 'BRKR' " + "broker, 'CLRM' clearing member. Exactly one RPTG row per UTI." + ) + role_sample_values = ["RPTG", "OTHR"] + + trade = relationship("Trade", back_populates="roles") + trade_relationship_description = "Trade this role row belongs to." + counterparty = relationship("Counterparty", back_populates="roles") + counterparty_relationship_description = "Entity playing this role." + + +@semantic_table( + description="Daily collateral/margin per UTI (Art. 11(3)).", + business_context=( + "SIGN CONVENTION (different from trade_state!): amounts are UNSIGNED; " + "direction is carried by which column they sit in. Net collateral = " + "posted - received, computed, never read from a column." + ), + time_dimension="reporting_date", +) +class CollateralReport(SemanticDeclarativeBase): + __tablename__ = "collateral_report" + + id = Column(Integer, primary_key=True) + uti = Column(String(52), ForeignKey("trade.uti"), nullable=False) + reporting_date = Column(Date, nullable=False) + reporting_date_time_grain = TimeGrain.DAY + + initial_margin_posted = Column(Numeric(20, 2)) + initial_margin_posted_description = "IM posted BY the reporting counterparty. Unsigned." + initial_margin_received = Column(Numeric(20, 2)) + initial_margin_received_description = ( + "IM received. Unsigned; do not sum with posted — subtract." + ) + collateral_currency = Column(String(3)) + collateral_currency_description = "ISO 4217 currency of margin amounts." + + +@semantic_table( + description=( + "Trade repository submission log. Grain: one row per submission " + "attempt — a trade may have many." + ), + business_context=( + "Rejection-rate questions resolve here, per attempt, not per trade, " + "unless deduplicated. Rejection rate = NACK / all attempts." + ), + time_dimension="submission_timestamp", +) +class SubmissionLog(SemanticDeclarativeBase): + __tablename__ = "submission_log" + + id = Column(Integer, primary_key=True) + uti = Column(String(52), ForeignKey("trade.uti"), nullable=False) + submission_timestamp = Column(DateTime, nullable=False) + submission_timestamp_time_grain = TimeGrain.SECOND + + status = Column(String(4), nullable=False) + status_description = "'ACKD' accepted, 'NACK' rejected, 'PEND' pending." + status_sample_values = ["ACKD", "NACK"] + + nack_reason = Column(String(200)) + nack_reason_description = "TR rejection reason text; NULL unless NACK." diff --git a/converters/semantido/tests/helpers.py b/converters/semantido/tests/helpers.py new file mode 100644 index 00000000..d7093dce --- /dev/null +++ b/converters/semantido/tests/helpers.py @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Shared test helpers.""" + +from pathlib import Path + +from ossie_semantido.loaders import load_from_module + +FIXTURES = Path(__file__).parent / "fixtures" + + +def load_emir_layer(): + return load_from_module( + "models.emir_reporting", sys_path=str(FIXTURES / "emir_reporting") + ) diff --git a/converters/semantido/tests/test_osi_to_semantido.py b/converters/semantido/tests/test_osi_to_semantido.py new file mode 100644 index 00000000..b03f8ad1 --- /dev/null +++ b/converters/semantido/tests/test_osi_to_semantido.py @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from ossie_semantido.osi_to_semantido import osi_to_semantido_source +from ossie_semantido.semantido_to_osi import semantic_layer_to_osi +from tests.helpers import load_emir_layer + + +def test_generated_source_compiles(): + layer = load_emir_layer() + document = semantic_layer_to_osi(layer, model_name="emir_reporting").output + result = osi_to_semantido_source(document) + compile(result.output, "", "exec") + + +def test_roundtrip_preserves_governance_annotations(): + layer = load_emir_layer() + document = semantic_layer_to_osi(layer, model_name="emir_reporting").output + source = osi_to_semantido_source(document).output + assert "sql_filters=" in source + assert "PrivacyLevel.CONFIDENTIAL" in source + assert 'time_dimension="reporting_date"' in source or "time_dimension='reporting_date'" in source + assert "TimeGrain.DAY" in source diff --git a/converters/semantido/tests/test_semantido_to_osi.py b/converters/semantido/tests/test_semantido_to_osi.py new file mode 100644 index 00000000..4b65a666 --- /dev/null +++ b/converters/semantido/tests/test_semantido_to_osi.py @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json + +from ossie import OSIDocument + +from ossie_semantido.semantido_to_osi import VENDOR_NAME, semantic_layer_to_osi +from tests.helpers import load_emir_layer + + +def _vendor_data(entity): + for ext in entity.custom_extensions or []: + if ext.vendor_name == VENDOR_NAME: + return json.loads(ext.data) + return {} + + +def test_converts_all_tables_to_datasets(): + layer = load_emir_layer() + result = semantic_layer_to_osi(layer, model_name="emir_reporting") + model = result.output.semantic_model[0] + assert {d.name for d in model.datasets} == set(layer.tables) + + +def test_relationships_are_parsed_not_dropped(): + layer = load_emir_layer() + result = semantic_layer_to_osi(layer, model_name="emir_reporting") + model = result.output.semantic_model[0] + assert model.relationships, "expected FK relationships in the fixture" + assert not result.issues, f"unexpected conversion issues: {result.issues}" + rel = model.relationships[0] + assert rel.from_columns and rel.to_columns + + +def test_governance_metadata_preserved_in_extensions(): + layer = load_emir_layer() + model = semantic_layer_to_osi(layer, model_name="emir_reporting").output.semantic_model[0] + + trade_state = next(d for d in model.datasets if d.name == "trade_state") + assert _vendor_data(trade_state).get("sql_filters"), "sql_filters must survive" + assert _vendor_data(trade_state).get("time_dimension") == "reporting_date" + + counterparty = next(d for d in model.datasets if d.name == "counterparty") + legal_name = next(f for f in counterparty.fields if f.name == "legal_name") + assert _vendor_data(legal_name).get("privacy_level") == "confidential" + + +def test_time_dimension_marked_on_field(): + layer = load_emir_layer() + model = semantic_layer_to_osi(layer, model_name="emir_reporting").output.semantic_model[0] + trade = next(d for d in model.datasets if d.name == "trade") + ts_field = next(f for f in trade.fields if f.name == "execution_timestamp") + assert ts_field.dimension is not None and ts_field.dimension.is_time is True + + +def test_yaml_output_is_valid_document(): + layer = load_emir_layer() + yaml_text = semantic_layer_to_osi(layer, model_name="emir_reporting").output.to_osi_yaml() + import yaml as pyyaml + + OSIDocument.model_validate(pyyaml.safe_load(yaml_text)) From bb04eb39cce988a20ca845ad2633b5ed2a4f5848 Mon Sep 17 00:00:00 2001 From: Dragos Crintea Date: Sat, 18 Jul 2026 23:15:25 +0100 Subject: [PATCH 3/5] semantido <-> ossie converter --- examples/semantido_to_osi_emir_reporting.yaml | 464 ++++++++++++++++++ 1 file changed, 464 insertions(+) create mode 100644 examples/semantido_to_osi_emir_reporting.yaml diff --git a/examples/semantido_to_osi_emir_reporting.yaml b/examples/semantido_to_osi_emir_reporting.yaml new file mode 100644 index 00000000..582334b3 --- /dev/null +++ b/examples/semantido_to_osi_emir_reporting.yaml @@ -0,0 +1,464 @@ +version: 0.2.0.dev0 +semantic_model: +- name: emir_reporting + datasets: + - name: collateral_report + source: collateral_report + primary_key: + - id + description: Daily collateral/margin per UTI (Art. 11(3)). + ai_context: + instructions: 'SIGN CONVENTION (different from trade_state!): amounts are UNSIGNED; + direction is carried by which column they sit in. Net collateral = posted + - received, computed, never read from a column.' + fields: + - name: id + expression: + dialects: + - dialect: ANSI_SQL + expression: id + description: 'Column: id' + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"data_type": "INTEGER"}' + - name: uti + expression: + dialects: + - dialect: ANSI_SQL + expression: uti + description: 'Column: uti' + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"data_type": "VARCHAR", "references": "trade.uti"}' + - name: reporting_date + expression: + dialects: + - dialect: ANSI_SQL + expression: reporting_date + dimension: + is_time: true + description: 'Column: reporting_date' + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"data_type": "DATE", "time_grain": "day"}' + - name: initial_margin_posted + expression: + dialects: + - dialect: ANSI_SQL + expression: initial_margin_posted + description: IM posted BY the reporting counterparty. Unsigned. + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"data_type": "DECIMAL"}' + - name: initial_margin_received + expression: + dialects: + - dialect: ANSI_SQL + expression: initial_margin_received + description: IM received. Unsigned; do not sum with posted — subtract. + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"data_type": "DECIMAL"}' + - name: collateral_currency + expression: + dialects: + - dialect: ANSI_SQL + expression: collateral_currency + description: ISO 4217 currency of margin amounts. + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"data_type": "VARCHAR"}' + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"time_dimension": "reporting_date"}' + - name: counterparty + source: counterparty + primary_key: + - lei + description: Legal entity master for derivative counterparties, one row per LEI. + ai_context: + instructions: EMIR counterparty classification drives clearing and reporting + obligations (FC / NFC+ / NFC- under Art. 2 and Art. 10). + synonyms: + - legal entities + - counterparties + fields: + - name: lei + expression: + dialects: + - dialect: ANSI_SQL + expression: lei + description: Legal Entity Identifier (ISO 17442). The sole join key for counterparty + identity everywhere in this schema. + ai_context: + examples: + - 529900T8BM49AURSDO55 + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"data_type": "VARCHAR"}' + - name: legal_name + expression: + dialects: + - dialect: ANSI_SQL + expression: legal_name + description: Registered legal name as per GLEIF record. + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"data_type": "VARCHAR", "privacy_level": "confidential"}' + - name: classification + expression: + dialects: + - dialect: ANSI_SQL + expression: classification + description: 'EMIR counterparty classification: ''FC'' (financial counterparty, + Art. 2(8)), ''NFC+'' (non-financial above clearing threshold, Art. 10), ''NFC-'' + (below). Filter on this before any obligation question.' + ai_context: + examples: + - FC + - NFC+ + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"data_type": "VARCHAR"}' + - name: country + expression: + dialects: + - dialect: ANSI_SQL + expression: country + description: ISO 3166-1 alpha-2 country of incorporation. + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"data_type": "VARCHAR"}' + - name: counterparty_trade_role + source: counterparty_trade_role + primary_key: + - id + description: Role bridge between trades and counterparties. A trade has AT LEAST + two rows here (reporting + other counterparty), often more. + ai_context: + instructions: 'FAN-OUT GUARD: joining trade -> this table -> counterparty multiplies + trade rows by role count; any SUM over trade amounts after this join double-counts. + Filter to a single role (usually ''RPTG'') before aggregating trade-level + amounts.' + fields: + - name: id + expression: + dialects: + - dialect: ANSI_SQL + expression: id + description: 'Column: id' + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"data_type": "INTEGER"}' + - name: uti + expression: + dialects: + - dialect: ANSI_SQL + expression: uti + description: 'Column: uti' + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"data_type": "VARCHAR", "references": "trade.uti"}' + - name: lei + expression: + dialects: + - dialect: ANSI_SQL + expression: lei + description: 'Column: lei' + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"data_type": "VARCHAR", "references": "counterparty.lei"}' + - name: role + expression: + dialects: + - dialect: ANSI_SQL + expression: role + description: '''RPTG'' reporting counterparty, ''OTHR'' other counterparty, + ''BRKR'' broker, ''CLRM'' clearing member. Exactly one RPTG row per UTI.' + ai_context: + examples: + - RPTG + - OTHR + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"data_type": "VARCHAR"}' + - name: submission_log + source: submission_log + primary_key: + - id + description: 'Trade repository submission log. Grain: one row per submission attempt + — a trade may have many.' + ai_context: + instructions: Rejection-rate questions resolve here, per attempt, not per trade, + unless deduplicated. Rejection rate = NACK / all attempts. + fields: + - name: id + expression: + dialects: + - dialect: ANSI_SQL + expression: id + description: 'Column: id' + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"data_type": "INTEGER"}' + - name: uti + expression: + dialects: + - dialect: ANSI_SQL + expression: uti + description: 'Column: uti' + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"data_type": "VARCHAR", "references": "trade.uti"}' + - name: submission_timestamp + expression: + dialects: + - dialect: ANSI_SQL + expression: submission_timestamp + dimension: + is_time: true + description: 'Column: submission_timestamp' + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"data_type": "TIMESTAMP", "time_grain": "second"}' + - name: status + expression: + dialects: + - dialect: ANSI_SQL + expression: status + description: '''ACKD'' accepted, ''NACK'' rejected, ''PEND'' pending.' + ai_context: + examples: + - ACKD + - NACK + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"data_type": "VARCHAR"}' + - name: nack_reason + expression: + dialects: + - dialect: ANSI_SQL + expression: nack_reason + description: TR rejection reason text; NULL unless NACK. + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"data_type": "VARCHAR"}' + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"time_dimension": "submission_timestamp"}' + - name: trade + source: trade + primary_key: + - uti + description: Reportable OTC derivative trades, one row per UTI. Immutable birth + record of the trade; evolving economics live in trade_state. + ai_context: + instructions: 'AMBIGUITY GUARD: notional_amount is contract size, NOT current + value and NOT exposure. ''How big'' -> notional; ''worth today'' -> trade_state.valuation_amount; + ''at risk'' -> collateral_report.' + synonyms: + - trades + - derivatives + - positions + fields: + - name: uti + expression: + dialects: + - dialect: ANSI_SQL + expression: uti + description: Unique Trade Identifier (Art. 9 REFIT). Canonical trade key. + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"data_type": "VARCHAR"}' + - name: asset_class + expression: + dialects: + - dialect: ANSI_SQL + expression: asset_class + description: 'EMIR asset class: ''IR'' rates, ''CR'' credit, ''EQ'' equity, + ''FX'' FX, ''CO'' commodity.' + ai_context: + examples: + - IR + - FX + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"data_type": "VARCHAR"}' + - name: notional_amount + expression: + dialects: + - dialect: ANSI_SQL + expression: notional_amount + description: Trade notional in notional_currency. Contract size, not value. + Never sum across currencies without conversion. + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"data_type": "DECIMAL", "privacy_level": "restricted"}' + - name: notional_currency + expression: + dialects: + - dialect: ANSI_SQL + expression: notional_currency + description: ISO 4217 currency of notional_amount. + ai_context: + examples: + - EUR + - USD + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"data_type": "VARCHAR"}' + - name: execution_timestamp + expression: + dialects: + - dialect: ANSI_SQL + expression: execution_timestamp + dimension: + is_time: true + description: UTC execution time (Art. 9 REFIT). Primary time axis. + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"data_type": "TIMESTAMP", "time_grain": "second"}' + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"time_dimension": "execution_timestamp"}' + - name: trade_state + source: trade_state + primary_key: + - id + description: Daily trade-state reports (EMIR REFIT is state-based), one row per + UTI per reporting_date. The latest state per UTI is the current view. + ai_context: + instructions: 'SIGN CONVENTION: valuation_amount is signed from the REPORTING + counterparty''s perspective. Aggregating across counterparties without flipping + signs by role is meaningless.' + fields: + - name: id + expression: + dialects: + - dialect: ANSI_SQL + expression: id + description: 'Column: id' + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"data_type": "INTEGER"}' + - name: uti + expression: + dialects: + - dialect: ANSI_SQL + expression: uti + description: 'Column: uti' + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"data_type": "VARCHAR", "references": "trade.uti"}' + - name: reporting_date + expression: + dialects: + - dialect: ANSI_SQL + expression: reporting_date + dimension: + is_time: true + description: State date; one state per trade per day. + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"data_type": "DATE", "time_grain": "day"}' + - name: valuation_amount + expression: + dialects: + - dialect: ANSI_SQL + expression: valuation_amount + description: 'Mark-to-market value (Art. 11(2)), signed from the reporting counterparty''s + perspective. Value, not size: do not confuse with notional.' + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"data_type": "DECIMAL", "privacy_level": "restricted"}' + - name: contract_status + expression: + dialects: + - dialect: ANSI_SQL + expression: contract_status + description: '''OUTS'' outstanding, ''TERM'' terminated, ''MATU'' matured, ''ERRO'' + error. Exclude non-OUTS from exposure aggregates unless asked otherwise.' + ai_context: + examples: + - OUTS + - TERM + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"data_type": "VARCHAR"}' + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"sql_filters": ["reporting_date = (SELECT MAX(ts2.reporting_date) FROM + trade_state ts2 WHERE ts2.uti = trade_state.uti)"], "time_dimension": "reporting_date"}' + relationships: + - name: counterparty_to_counterparty_trade_role + from: counterparty + to: counterparty_trade_role + from_columns: + - lei + to_columns: + - lei + ai_context: + instructions: Roles this entity plays on trades. + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"relationship_type": "one-to-many"}' + - name: counterparty_trade_role_to_trade + from: counterparty_trade_role + to: trade + from_columns: + - uti + to_columns: + - uti + ai_context: + instructions: Trade this role row belongs to. + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"relationship_type": "many-to-one"}' + - name: counterparty_trade_role_to_counterparty + from: counterparty_trade_role + to: counterparty + from_columns: + - lei + to_columns: + - lei + ai_context: + instructions: Entity playing this role. + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"relationship_type": "many-to-one"}' + - name: trade_to_trade_state + from: trade + to: trade_state + from_columns: + - uti + to_columns: + - uti + ai_context: + instructions: Daily state reports for this trade. + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"relationship_type": "one-to-many"}' + - name: trade_to_counterparty_trade_role + from: trade + to: counterparty_trade_role + from_columns: + - uti + to_columns: + - uti + ai_context: + instructions: Counterparty roles attached to this trade. + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"relationship_type": "one-to-many"}' + - name: trade_state_to_trade + from: trade_state + to: trade + from_columns: + - uti + to_columns: + - uti + ai_context: + instructions: The trade this state report belongs to. + custom_extensions: + - vendor_name: SEMANTIDO + data: '{"relationship_type": "many-to-one"}' From aecd2119ead1cefded79c522994e7c692a99f822 Mon Sep 17 00:00:00 2001 From: Dragos Crintea Date: Sat, 18 Jul 2026 23:26:43 +0100 Subject: [PATCH 4/5] semantido <-> ossie converter --- examples/semantido_to_osi_emir_reporting.yaml | 464 ------------------ 1 file changed, 464 deletions(-) delete mode 100644 examples/semantido_to_osi_emir_reporting.yaml diff --git a/examples/semantido_to_osi_emir_reporting.yaml b/examples/semantido_to_osi_emir_reporting.yaml deleted file mode 100644 index 582334b3..00000000 --- a/examples/semantido_to_osi_emir_reporting.yaml +++ /dev/null @@ -1,464 +0,0 @@ -version: 0.2.0.dev0 -semantic_model: -- name: emir_reporting - datasets: - - name: collateral_report - source: collateral_report - primary_key: - - id - description: Daily collateral/margin per UTI (Art. 11(3)). - ai_context: - instructions: 'SIGN CONVENTION (different from trade_state!): amounts are UNSIGNED; - direction is carried by which column they sit in. Net collateral = posted - - received, computed, never read from a column.' - fields: - - name: id - expression: - dialects: - - dialect: ANSI_SQL - expression: id - description: 'Column: id' - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"data_type": "INTEGER"}' - - name: uti - expression: - dialects: - - dialect: ANSI_SQL - expression: uti - description: 'Column: uti' - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"data_type": "VARCHAR", "references": "trade.uti"}' - - name: reporting_date - expression: - dialects: - - dialect: ANSI_SQL - expression: reporting_date - dimension: - is_time: true - description: 'Column: reporting_date' - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"data_type": "DATE", "time_grain": "day"}' - - name: initial_margin_posted - expression: - dialects: - - dialect: ANSI_SQL - expression: initial_margin_posted - description: IM posted BY the reporting counterparty. Unsigned. - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"data_type": "DECIMAL"}' - - name: initial_margin_received - expression: - dialects: - - dialect: ANSI_SQL - expression: initial_margin_received - description: IM received. Unsigned; do not sum with posted — subtract. - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"data_type": "DECIMAL"}' - - name: collateral_currency - expression: - dialects: - - dialect: ANSI_SQL - expression: collateral_currency - description: ISO 4217 currency of margin amounts. - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"data_type": "VARCHAR"}' - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"time_dimension": "reporting_date"}' - - name: counterparty - source: counterparty - primary_key: - - lei - description: Legal entity master for derivative counterparties, one row per LEI. - ai_context: - instructions: EMIR counterparty classification drives clearing and reporting - obligations (FC / NFC+ / NFC- under Art. 2 and Art. 10). - synonyms: - - legal entities - - counterparties - fields: - - name: lei - expression: - dialects: - - dialect: ANSI_SQL - expression: lei - description: Legal Entity Identifier (ISO 17442). The sole join key for counterparty - identity everywhere in this schema. - ai_context: - examples: - - 529900T8BM49AURSDO55 - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"data_type": "VARCHAR"}' - - name: legal_name - expression: - dialects: - - dialect: ANSI_SQL - expression: legal_name - description: Registered legal name as per GLEIF record. - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"data_type": "VARCHAR", "privacy_level": "confidential"}' - - name: classification - expression: - dialects: - - dialect: ANSI_SQL - expression: classification - description: 'EMIR counterparty classification: ''FC'' (financial counterparty, - Art. 2(8)), ''NFC+'' (non-financial above clearing threshold, Art. 10), ''NFC-'' - (below). Filter on this before any obligation question.' - ai_context: - examples: - - FC - - NFC+ - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"data_type": "VARCHAR"}' - - name: country - expression: - dialects: - - dialect: ANSI_SQL - expression: country - description: ISO 3166-1 alpha-2 country of incorporation. - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"data_type": "VARCHAR"}' - - name: counterparty_trade_role - source: counterparty_trade_role - primary_key: - - id - description: Role bridge between trades and counterparties. A trade has AT LEAST - two rows here (reporting + other counterparty), often more. - ai_context: - instructions: 'FAN-OUT GUARD: joining trade -> this table -> counterparty multiplies - trade rows by role count; any SUM over trade amounts after this join double-counts. - Filter to a single role (usually ''RPTG'') before aggregating trade-level - amounts.' - fields: - - name: id - expression: - dialects: - - dialect: ANSI_SQL - expression: id - description: 'Column: id' - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"data_type": "INTEGER"}' - - name: uti - expression: - dialects: - - dialect: ANSI_SQL - expression: uti - description: 'Column: uti' - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"data_type": "VARCHAR", "references": "trade.uti"}' - - name: lei - expression: - dialects: - - dialect: ANSI_SQL - expression: lei - description: 'Column: lei' - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"data_type": "VARCHAR", "references": "counterparty.lei"}' - - name: role - expression: - dialects: - - dialect: ANSI_SQL - expression: role - description: '''RPTG'' reporting counterparty, ''OTHR'' other counterparty, - ''BRKR'' broker, ''CLRM'' clearing member. Exactly one RPTG row per UTI.' - ai_context: - examples: - - RPTG - - OTHR - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"data_type": "VARCHAR"}' - - name: submission_log - source: submission_log - primary_key: - - id - description: 'Trade repository submission log. Grain: one row per submission attempt - — a trade may have many.' - ai_context: - instructions: Rejection-rate questions resolve here, per attempt, not per trade, - unless deduplicated. Rejection rate = NACK / all attempts. - fields: - - name: id - expression: - dialects: - - dialect: ANSI_SQL - expression: id - description: 'Column: id' - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"data_type": "INTEGER"}' - - name: uti - expression: - dialects: - - dialect: ANSI_SQL - expression: uti - description: 'Column: uti' - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"data_type": "VARCHAR", "references": "trade.uti"}' - - name: submission_timestamp - expression: - dialects: - - dialect: ANSI_SQL - expression: submission_timestamp - dimension: - is_time: true - description: 'Column: submission_timestamp' - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"data_type": "TIMESTAMP", "time_grain": "second"}' - - name: status - expression: - dialects: - - dialect: ANSI_SQL - expression: status - description: '''ACKD'' accepted, ''NACK'' rejected, ''PEND'' pending.' - ai_context: - examples: - - ACKD - - NACK - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"data_type": "VARCHAR"}' - - name: nack_reason - expression: - dialects: - - dialect: ANSI_SQL - expression: nack_reason - description: TR rejection reason text; NULL unless NACK. - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"data_type": "VARCHAR"}' - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"time_dimension": "submission_timestamp"}' - - name: trade - source: trade - primary_key: - - uti - description: Reportable OTC derivative trades, one row per UTI. Immutable birth - record of the trade; evolving economics live in trade_state. - ai_context: - instructions: 'AMBIGUITY GUARD: notional_amount is contract size, NOT current - value and NOT exposure. ''How big'' -> notional; ''worth today'' -> trade_state.valuation_amount; - ''at risk'' -> collateral_report.' - synonyms: - - trades - - derivatives - - positions - fields: - - name: uti - expression: - dialects: - - dialect: ANSI_SQL - expression: uti - description: Unique Trade Identifier (Art. 9 REFIT). Canonical trade key. - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"data_type": "VARCHAR"}' - - name: asset_class - expression: - dialects: - - dialect: ANSI_SQL - expression: asset_class - description: 'EMIR asset class: ''IR'' rates, ''CR'' credit, ''EQ'' equity, - ''FX'' FX, ''CO'' commodity.' - ai_context: - examples: - - IR - - FX - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"data_type": "VARCHAR"}' - - name: notional_amount - expression: - dialects: - - dialect: ANSI_SQL - expression: notional_amount - description: Trade notional in notional_currency. Contract size, not value. - Never sum across currencies without conversion. - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"data_type": "DECIMAL", "privacy_level": "restricted"}' - - name: notional_currency - expression: - dialects: - - dialect: ANSI_SQL - expression: notional_currency - description: ISO 4217 currency of notional_amount. - ai_context: - examples: - - EUR - - USD - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"data_type": "VARCHAR"}' - - name: execution_timestamp - expression: - dialects: - - dialect: ANSI_SQL - expression: execution_timestamp - dimension: - is_time: true - description: UTC execution time (Art. 9 REFIT). Primary time axis. - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"data_type": "TIMESTAMP", "time_grain": "second"}' - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"time_dimension": "execution_timestamp"}' - - name: trade_state - source: trade_state - primary_key: - - id - description: Daily trade-state reports (EMIR REFIT is state-based), one row per - UTI per reporting_date. The latest state per UTI is the current view. - ai_context: - instructions: 'SIGN CONVENTION: valuation_amount is signed from the REPORTING - counterparty''s perspective. Aggregating across counterparties without flipping - signs by role is meaningless.' - fields: - - name: id - expression: - dialects: - - dialect: ANSI_SQL - expression: id - description: 'Column: id' - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"data_type": "INTEGER"}' - - name: uti - expression: - dialects: - - dialect: ANSI_SQL - expression: uti - description: 'Column: uti' - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"data_type": "VARCHAR", "references": "trade.uti"}' - - name: reporting_date - expression: - dialects: - - dialect: ANSI_SQL - expression: reporting_date - dimension: - is_time: true - description: State date; one state per trade per day. - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"data_type": "DATE", "time_grain": "day"}' - - name: valuation_amount - expression: - dialects: - - dialect: ANSI_SQL - expression: valuation_amount - description: 'Mark-to-market value (Art. 11(2)), signed from the reporting counterparty''s - perspective. Value, not size: do not confuse with notional.' - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"data_type": "DECIMAL", "privacy_level": "restricted"}' - - name: contract_status - expression: - dialects: - - dialect: ANSI_SQL - expression: contract_status - description: '''OUTS'' outstanding, ''TERM'' terminated, ''MATU'' matured, ''ERRO'' - error. Exclude non-OUTS from exposure aggregates unless asked otherwise.' - ai_context: - examples: - - OUTS - - TERM - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"data_type": "VARCHAR"}' - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"sql_filters": ["reporting_date = (SELECT MAX(ts2.reporting_date) FROM - trade_state ts2 WHERE ts2.uti = trade_state.uti)"], "time_dimension": "reporting_date"}' - relationships: - - name: counterparty_to_counterparty_trade_role - from: counterparty - to: counterparty_trade_role - from_columns: - - lei - to_columns: - - lei - ai_context: - instructions: Roles this entity plays on trades. - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"relationship_type": "one-to-many"}' - - name: counterparty_trade_role_to_trade - from: counterparty_trade_role - to: trade - from_columns: - - uti - to_columns: - - uti - ai_context: - instructions: Trade this role row belongs to. - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"relationship_type": "many-to-one"}' - - name: counterparty_trade_role_to_counterparty - from: counterparty_trade_role - to: counterparty - from_columns: - - lei - to_columns: - - lei - ai_context: - instructions: Entity playing this role. - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"relationship_type": "many-to-one"}' - - name: trade_to_trade_state - from: trade - to: trade_state - from_columns: - - uti - to_columns: - - uti - ai_context: - instructions: Daily state reports for this trade. - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"relationship_type": "one-to-many"}' - - name: trade_to_counterparty_trade_role - from: trade - to: counterparty_trade_role - from_columns: - - uti - to_columns: - - uti - ai_context: - instructions: Counterparty roles attached to this trade. - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"relationship_type": "one-to-many"}' - - name: trade_state_to_trade - from: trade_state - to: trade - from_columns: - - uti - to_columns: - - uti - ai_context: - instructions: The trade this state report belongs to. - custom_extensions: - - vendor_name: SEMANTIDO - data: '{"relationship_type": "many-to-one"}' From 17992f9b41011681040dd746b0f3b4739657145f Mon Sep 17 00:00:00 2001 From: Dragos Crintea Date: Tue, 21 Jul 2026 19:04:33 +0100 Subject: [PATCH 5/5] resolved raised PR request comments --- converters/semantido/pyproject.toml | 3 +- .../semantido/src/ossie_semantido/__init__.py | 6 +- .../semantido/src/ossie_semantido/cli.py | 54 ++++++++++++----- .../src/ossie_semantido/constants.py | 20 +++++++ .../src/ossie_semantido/osi_to_semantido.py | 25 +++++--- .../src/ossie_semantido/semantido_to_osi.py | 58 ++++++++++++++----- 6 files changed, 129 insertions(+), 37 deletions(-) create mode 100644 converters/semantido/src/ossie_semantido/constants.py diff --git a/converters/semantido/pyproject.toml b/converters/semantido/pyproject.toml index 89f5d54e..a2d27d39 100644 --- a/converters/semantido/pyproject.toml +++ b/converters/semantido/pyproject.toml @@ -30,7 +30,7 @@ classifiers = [ ] dependencies = [ "apache-ossie>=0.2.0.dev0", - "semantido>=0.4.0", + "semantido>=0.4.1", "SQLAlchemy>=2.0", "PyYAML>=6.0", ] @@ -47,6 +47,7 @@ packages = ["src/ossie_semantido"] [tool.uv] dev-dependencies = [ "pytest>=8.0", + "ruff==0.15.22" ] [tool.uv.sources] diff --git a/converters/semantido/src/ossie_semantido/__init__.py b/converters/semantido/src/ossie_semantido/__init__.py index 8a4f9377..1014e6f9 100644 --- a/converters/semantido/src/ossie_semantido/__init__.py +++ b/converters/semantido/src/ossie_semantido/__init__.py @@ -17,7 +17,11 @@ """semantido <> Apache Ossie converter.""" -from ossie_semantido.converter_issues import ConverterIssue, ConverterIssueType, ConverterResult +from ossie_semantido.converter_issues import ( + ConverterIssue, + ConverterIssueType, + ConverterResult, +) from ossie_semantido.loaders import load_from_module from ossie_semantido.osi_to_semantido import osi_to_semantido_source from ossie_semantido.semantido_to_osi import semantic_layer_to_osi diff --git a/converters/semantido/src/ossie_semantido/cli.py b/converters/semantido/src/ossie_semantido/cli.py index 7e26df68..138cbd26 100644 --- a/converters/semantido/src/ossie_semantido/cli.py +++ b/converters/semantido/src/ossie_semantido/cli.py @@ -30,7 +30,9 @@ def _report_issues(issues) -> None: for issue in issues: - print(f"warning: {issue.issue_type.value}: {issue.element_name}", file=sys.stderr) + print( + f"warning: {issue.issue_type.value}: {issue.element_name}", file=sys.stderr + ) def _cmd_semantido_to_osi(args: argparse.Namespace) -> None: @@ -60,23 +62,49 @@ def main() -> None: to_osi = subparsers.add_parser( "semantido-to-osi", help="Convert decorated SQLAlchemy models → Ossie YAML" ) - to_osi.add_argument("-m", "--module", required=True, metavar="MODULE", - help="Dotted module path of the semantido models, e.g. models.emir_reporting") - to_osi.add_argument("-p", "--path", default=None, metavar="DIR", - help="Directory prepended to sys.path before import") - to_osi.add_argument("-n", "--name", required=True, metavar="NAME", - help="Name for the Ossie semantic_model") - to_osi.add_argument("-o", "--output", required=True, metavar="FILE", - help="Path for output Ossie YAML") + to_osi.add_argument( + "-m", + "--module", + required=True, + metavar="MODULE", + help="Dotted module path of the semantido models, e.g. models.emir_reporting", + ) + to_osi.add_argument( + "-p", + "--path", + default=None, + metavar="DIR", + help="Directory prepended to sys.path before import", + ) + to_osi.add_argument( + "-n", + "--name", + required=True, + metavar="NAME", + help="Name for the Ossie semantic_model", + ) + to_osi.add_argument( + "-o", + "--output", + required=True, + metavar="FILE", + help="Path for output Ossie YAML", + ) to_osi.set_defaults(func=_cmd_semantido_to_osi) from_osi = subparsers.add_parser( "osi-to-semantido", help="Convert Ossie YAML → generated semantido model code" ) - from_osi.add_argument("-i", "--input", required=True, metavar="FILE", - help="Path to Ossie YAML") - from_osi.add_argument("-o", "--output", required=True, metavar="FILE", - help="Path for generated Python module") + from_osi.add_argument( + "-i", "--input", required=True, metavar="FILE", help="Path to Ossie YAML" + ) + from_osi.add_argument( + "-o", + "--output", + required=True, + metavar="FILE", + help="Path for generated Python module", + ) from_osi.set_defaults(func=_cmd_osi_to_semantido) args = parser.parse_args() diff --git a/converters/semantido/src/ossie_semantido/constants.py b/converters/semantido/src/ossie_semantido/constants.py new file mode 100644 index 00000000..5e07cbb1 --- /dev/null +++ b/converters/semantido/src/ossie_semantido/constants.py @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Shared constants for the ossie-semantido converter.""" + +VENDOR_NAME = "SEMANTIDO" diff --git a/converters/semantido/src/ossie_semantido/osi_to_semantido.py b/converters/semantido/src/ossie_semantido/osi_to_semantido.py index 91a4aa2c..8aa9eb0c 100644 --- a/converters/semantido/src/ossie_semantido/osi_to_semantido.py +++ b/converters/semantido/src/ossie_semantido/osi_to_semantido.py @@ -29,9 +29,12 @@ from ossie import OSIDataset, OSIDocument, OSIField -from ossie_semantido.converter_issues import ConverterIssue, ConverterIssueType, ConverterResult - -VENDOR_NAME = "SEMANTIDO" +from ossie_semantido.constants import VENDOR_NAME +from ossie_semantido.converter_issues import ( + ConverterIssue, + ConverterIssueType, + ConverterResult, +) _TYPE_MAP = { "VARCHAR": "String", @@ -67,7 +70,9 @@ def _sqlalchemy_type(field: OSIField, issues: List[ConverterIssue]) -> str: return _TYPE_MAP[base] if base: issues.append( - ConverterIssue(ConverterIssueType.UNMAPPED_DATA_TYPE, f"{field.name}:{base}") + ConverterIssue( + ConverterIssueType.UNMAPPED_DATA_TYPE, f"{field.name}:{base}" + ) ) return "String" @@ -117,7 +122,9 @@ def _emit_dataset(dataset: OSIDataset, issues: List[ConverterIssue]) -> str: level = field_payload["privacy_level"].upper() lines.append(f" {field.name}_privacy_level = PrivacyLevel.{level}") if field_ctx and field_ctx.examples: - lines.append(f" {field.name}_sample_values = {[str(e) for e in field_ctx.examples]!r}") + lines.append( + f" {field.name}_sample_values = {[str(e) for e in field_ctx.examples]!r}" + ) if field_payload.get("time_grain"): grain = field_payload["time_grain"].upper() lines.append(f" {field.name}_time_grain = TimeGrain.{grain}") @@ -131,13 +138,17 @@ def osi_to_semantido_source(document: OSIDocument) -> ConverterResult[str]: for model in document.semantic_model: for metric in model.metrics or []: - issues.append(ConverterIssue(ConverterIssueType.METRIC_DROPPED, metric.name)) + issues.append( + ConverterIssue(ConverterIssueType.METRIC_DROPPED, metric.name) + ) for dataset in model.datasets: blocks.append(_emit_dataset(dataset, issues)) todo_block = "" if issues: - todo_lines = "\n".join(f"# {i.issue_type.value}: {i.element_name}" for i in issues) + todo_lines = "\n".join( + f"# {i.issue_type.value}: {i.element_name}" for i in issues + ) todo_block = ( "# TODO(ossie-semantido): the following Ossie constructs have no\n" "# semantido equivalent yet and were not converted:\n" + todo_lines + "\n\n" diff --git a/converters/semantido/src/ossie_semantido/semantido_to_osi.py b/converters/semantido/src/ossie_semantido/semantido_to_osi.py index 74743b85..9cf55689 100644 --- a/converters/semantido/src/ossie_semantido/semantido_to_osi.py +++ b/converters/semantido/src/ossie_semantido/semantido_to_osi.py @@ -17,7 +17,7 @@ """semantido SemanticLayer -> Apache Ossie document conversion. -Builds the Ossie model through the typed ``apache-ossie`` objects so the +Builds the Ossie model through the typed ``apache-ossie`` objects, so the output is schema-conformant by construction. Governance metadata that has no Ossie core field (privacy levels, SQL filters, time grain, the primary time axis) is preserved losslessly in ``custom_extensions`` under the @@ -42,16 +42,22 @@ OSIRelationship, OSISemanticModel, ) -from semantido.generators.semantic_layer import Column, Relationship, SemanticLayer, Table - -from ossie_semantido.converter_issues import ConverterIssue, ConverterIssueType, ConverterResult - -VENDOR_NAME = "SEMANTIDO" +from semantido.generators.semantic_layer import ( + Column, + Relationship, + SemanticLayer, + Table, +) -_JOIN_RE = re.compile( - r"^\s*(\w+)\.(\w+)\s*=\s*(\w+)\.(\w+)\s*$" +from ossie_semantido.constants import VENDOR_NAME +from ossie_semantido.converter_issues import ( + ConverterIssue, + ConverterIssueType, + ConverterResult, ) +_JOIN_RE = re.compile(r"^\s*(\w+)\.(\w+)\s*=\s*(\w+)\.(\w+)\s*$") + def _extension(payload: dict) -> OSICustomExtension: """Wrap semantido-specific metadata as a spec-conformant custom extension.""" @@ -63,7 +69,9 @@ def _extension(payload: dict) -> OSICustomExtension: def _ansi(expression: str) -> OSIExpression: return OSIExpression( - dialects=[OSIDialectExpression(dialect=OSIDialect.ANSI_SQL, expression=expression)] + dialects=[ + OSIDialectExpression(dialect=OSIDialect.ANSI_SQL, expression=expression) + ] ) @@ -92,12 +100,16 @@ def _column_to_field(column: Column) -> OSIField: dimension=OSIDimension(is_time=True) if column.is_time_dimension else None, description=column.description or None, ai_context=OSIAIContextObject(**ai_kwargs) if ai_kwargs else None, - custom_extensions=[_extension(extension_payload)] if extension_payload else None, + custom_extensions=[_extension(extension_payload)] + if extension_payload + else None, ) def _table_to_dataset(table: Table) -> OSIDataset: - instructions_parts = [p for p in (table.business_context, table.application_context) if p] + instructions_parts = [ + p for p in (table.business_context, table.application_context) if p + ] ai_kwargs = {} if instructions_parts: @@ -118,7 +130,9 @@ def _table_to_dataset(table: Table) -> OSIDataset: description=table.description or None, ai_context=OSIAIContextObject(**ai_kwargs) if ai_kwargs else None, fields=[_column_to_field(c) for c in table.columns] or None, - custom_extensions=[_extension(extension_payload)] if extension_payload else None, + custom_extensions=[_extension(extension_payload)] + if extension_payload + else None, ) @@ -140,8 +154,18 @@ def _relationship_to_osi( # which side of the equality each table appeared on. if left_table == rel.from_table: from_columns, to_columns = [left_col], [right_col] - else: + elif right_table == rel.from_table: from_columns, to_columns = [right_col], [left_col] + else: + # Neither side of the join references rel.from_table (e.g., aliases) + # Treat this as unparseable rather than silently swapping columns. + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.RELATIONSHIP_JOIN_UNPARSED, + element_name=f"{rel.from_table}->{rel.to_table}", + ) + ) + return None ai_kwargs = {} if rel.description: @@ -156,7 +180,9 @@ def _relationship_to_osi( "to_columns": to_columns, "ai_context": ai_kwargs or None, "custom_extensions": [ - _extension({"relationship_type": rel.relationship_type.value}).model_dump() + _extension( + {"relationship_type": rel.relationship_type.value} + ).model_dump() ], } ) @@ -178,7 +204,9 @@ def semantic_layer_to_osi( model_extensions = None if layer.application_glossary: - model_extensions = [_extension({"application_glossary": layer.application_glossary})] + model_extensions = [ + _extension({"application_glossary": layer.application_glossary}) + ] model = OSISemanticModel( name=model_name,