diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index a02f39d..b0e548f 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -5,9 +5,9 @@ {"id":"openforge_catalog-5gb","title":"Frontend: Admin camera position controls UI","description":"Build admin interface to preview and set default camera angles for models. Shows live preview of each angle, allows selecting which is default. Integrates with admin set camera position API. Consider batch operations for setting defaults across similar models.","status":"open","priority":2,"issue_type":"task","created_at":"2025-12-21T09:58:45.023938263-07:00","updated_at":"2025-12-21T09:58:45.023938263-07:00","dependencies":[{"issue_id":"openforge_catalog-5gb","depends_on_id":"openforge_catalog-bll","type":"parent-child","created_at":"2025-12-21T09:59:19.862145404-07:00","created_by":"daemon"},{"issue_id":"openforge_catalog-5gb","depends_on_id":"openforge_catalog-hqe","type":"blocks","created_at":"2025-12-21T09:59:36.561390068-07:00","created_by":"daemon"}]} {"id":"openforge_catalog-5na","title":"Phased rollout: Test subset before bulk backfill","description":"Implement safe migration strategy: (1) Generate sprite sheets for small test set (~10-20 diverse models). (2) Deploy frontend with fallback to legacy single thumbnails. (3) Validate with users - test interactive controls, image quality, performance. (4) Iterate on camera positions/layout based on feedback. (5) Only after validation, run bulk backfill for all 8000+ STL files. Include ability to pause/resume bulk processing. Track which models have sprite sheets vs legacy thumbnails.","status":"open","priority":2,"issue_type":"task","created_at":"2025-12-21T10:13:57.517770238-07:00","updated_at":"2025-12-21T10:13:57.517770238-07:00","dependencies":[{"issue_id":"openforge_catalog-5na","depends_on_id":"openforge_catalog-bll","type":"parent-child","created_at":"2025-12-21T10:14:18.760726353-07:00","created_by":"daemon"},{"issue_id":"openforge_catalog-5na","depends_on_id":"openforge_catalog-4ji","type":"blocks","created_at":"2025-12-21T10:14:23.673079682-07:00","created_by":"daemon"},{"issue_id":"openforge_catalog-5na","depends_on_id":"openforge_catalog-3cx","type":"blocks","created_at":"2025-12-21T10:14:24.071569003-07:00","created_by":"daemon"},{"issue_id":"openforge_catalog-5na","depends_on_id":"openforge_catalog-dd4","type":"blocks","created_at":"2025-12-21T10:14:24.494018624-07:00","created_by":"daemon"}]} {"id":"openforge_catalog-5tg","title":"Backend API: Get thumbnail variants endpoint","description":"Create Flask endpoint to return sprite sheet info for a model. Returns: (1) Sprite sheet URL, (2) Grid layout (rows, cols, tile dimensions), (3) Angle metadata (name, position in grid, camera coords), (4) Default angle index. Frontend uses this to display correct region from sprite sheet. Handles models with legacy single thumbnail gracefully.","status":"closed","priority":2,"issue_type":"task","created_at":"2025-12-21T09:58:32.287223899-07:00","updated_at":"2025-12-21T14:00:25.231595972-07:00","closed_at":"2025-12-21T14:00:25.231595972-07:00","close_reason":"Completed: Created GET /api/blueprints/\u003cblueprint_id\u003e/thumbnail-variants endpoint. Returns sprite sheet URL, grid layout (rows/cols/tile_size), angle metadata with camera positions, and default angle index. Gracefully handles legacy single thumbnails. All 319 tests passing.","dependencies":[{"issue_id":"openforge_catalog-5tg","depends_on_id":"openforge_catalog-bll","type":"parent-child","created_at":"2025-12-21T09:59:18.52350897-07:00","created_by":"daemon"},{"issue_id":"openforge_catalog-5tg","depends_on_id":"openforge_catalog-ubp","type":"blocks","created_at":"2025-12-21T09:59:35.26824895-07:00","created_by":"daemon"}]} -{"id":"openforge_catalog-750","title":"Postgres migration: thingiverse sync-state tables","description":"New schema version adding: thingiverse_thing (blueprint_id \u003c-\u003e thing_id, status, published_at, last_synced_at) and thingiverse_file (thing ref, catalog file ref + local hash, remote file id + remote hash, last_synced_at) — the diff ledger for the sync engine. Follow existing openforge/db/schema/version_NN.py migration pattern. If the HAR ticket proves the remote hash is SHA-256 (catalog stores MD5), include storage for the second hash algorithm.","notes":"SCOPE NOTES: (1) thingiverse_file must carry a file-type/role discriminator (model/image/zip/other) since things contain mixed file types. (2) This runs against the LOCAL db; prod gets a subset via the fixture path (see openforge_catalog-8yb) — keep the mapping (blueprint_id, thing_id, public_url) cleanly separable from local-only operational state (hash ledger, sync timestamps).","status":"open","priority":2,"issue_type":"task","owner":"devon.jones@gmail.com","created_at":"2026-07-06T11:42:25.104764326-06:00","created_by":"Devon Jones","updated_at":"2026-07-06T11:55:49.425508213-06:00","dependencies":[{"issue_id":"openforge_catalog-750","depends_on_id":"openforge_catalog-4kx","type":"parent-child","created_at":"2026-07-06T11:42:52.476138903-06:00","created_by":"Devon Jones"}]} +{"id":"openforge_catalog-750","title":"Postgres migration: thingiverse sync-state tables","description":"New schema version adding: thingiverse_thing (blueprint_id \u003c-\u003e thing_id, status, published_at, last_synced_at) and thingiverse_file (thing ref, catalog file ref + local hash, remote file id + remote hash, last_synced_at) — the diff ledger for the sync engine. Follow existing openforge/db/schema/version_NN.py migration pattern. If the HAR ticket proves the remote hash is SHA-256 (catalog stores MD5), include storage for the second hash algorithm.","notes":"SCOPE NOTES: (1) thingiverse_file must carry a file-type/role discriminator (model/image/zip/other) since things contain mixed file types. (2) This runs against the LOCAL db; prod gets a subset via the fixture path (see openforge_catalog-8yb) — keep the mapping (blueprint_id, thing_id, public_url) cleanly separable from local-only operational state (hash ledger, sync timestamps).","status":"in_progress","priority":2,"issue_type":"task","owner":"devon.jones@gmail.com","created_at":"2026-07-06T11:42:25.104764326-06:00","created_by":"Devon Jones","updated_at":"2026-07-06T14:40:56.467812707-06:00","dependencies":[{"issue_id":"openforge_catalog-750","depends_on_id":"openforge_catalog-4kx","type":"parent-child","created_at":"2026-07-06T11:42:52.476138903-06:00","created_by":"Devon Jones"}]} {"id":"openforge_catalog-7dg","title":"Auth/token manager for Thingiverse v2 (login + refresh)","description":"Python module: interactive 'login' does POST /v2/auth/login {usernameOrEmail, password} -\u003e AuthTokensResponse (handle 202 2FA via /v2/auth/2fa/login), persists refresh token securely (env file/keyring, never in repo); runtime path auto-refreshes access JWT via POST /v2/auth/refresh. Password is never stored. Buildable and testable today against the live API — verify a Devon-scoped JWT via GET /v2/users/me. Note: THINGIVERSE_APP_TOKEN in ~/.profile.d/099.api_keys.sh is v1-only and scoped to the wrong account (Joaquin Munguia, id 5577639) — do not use.","status":"closed","priority":1,"issue_type":"feature","owner":"devon.jones@gmail.com","created_at":"2026-07-06T11:42:24.708129858-06:00","created_by":"Devon Jones","updated_at":"2026-07-06T14:39:30.409750275-06:00","closed_at":"2026-07-06T14:39:30.409750275-06:00","close_reason":"Merged in PR #224 to test. TokenManager (login/2FA/refresh/atomic 0600 persistence/server-side logout) + tv_auth CLI + 53 tests at 100% coverage. Live-verified: devonjones (id 47139) login, status, and refresh rotation against the real v2 API. Unblocks gn2 together with hnr (HAR capture).","dependencies":[{"issue_id":"openforge_catalog-7dg","depends_on_id":"openforge_catalog-4kx","type":"parent-child","created_at":"2026-07-06T11:42:52.071225891-06:00","created_by":"Devon Jones"}]} -{"id":"openforge_catalog-8yb","title":"Propagate thingiverse sync data to AWS prod (fixture + API upsert path)","description":"The tool runs against the LOCAL Postgres, but prod (AWS Lambda + serverless Postgres) is populated by replaying fixture JSON through the Flask API (bin/upload_fixture with OPENFORGE_BASE_URL, same pattern as blueprint fixtures). Thingiverse data (blueprint_id \u003c-\u003e thing_id mapping, public thing URLs) must ride that path so the app can render Thingiverse links in prod and support deeper integration later. Deliverables: fixture format for thingiverse mappings (exported from the local sync tables), an authenticated API upsert endpoint in the Flask app, and upload_fixture support. Note: prod likely only needs the mapping/links, not the per-file hash ledger — that is local operational state; decide the split when designing the fixture.","status":"open","priority":2,"issue_type":"feature","owner":"devon.jones@gmail.com","created_at":"2026-07-06T11:55:33.474857151-06:00","created_by":"Devon Jones","updated_at":"2026-07-06T11:55:33.474857151-06:00","dependencies":[{"issue_id":"openforge_catalog-8yb","depends_on_id":"openforge_catalog-750","type":"blocks","created_at":"2026-07-06T11:55:47.512062985-06:00","created_by":"Devon Jones"},{"issue_id":"openforge_catalog-8yb","depends_on_id":"openforge_catalog-4kx","type":"parent-child","created_at":"2026-07-06T11:55:47.882392741-06:00","created_by":"Devon Jones"}]} +{"id":"openforge_catalog-8yb","title":"Propagate thingiverse sync data to AWS prod (fixture + API upsert path)","description":"The tool runs against the LOCAL Postgres, but prod (AWS Lambda + serverless Postgres) is populated by replaying fixture JSON through the Flask API (bin/upload_fixture with OPENFORGE_BASE_URL, same pattern as blueprint fixtures). Thingiverse data (blueprint_id \u003c-\u003e thing_id mapping, public thing URLs) must ride that path so the app can render Thingiverse links in prod and support deeper integration later. Deliverables: fixture format for thingiverse mappings (exported from the local sync tables), an authenticated API upsert endpoint in the Flask app, and upload_fixture support. Note: prod likely only needs the mapping/links, not the per-file hash ledger — that is local operational state; decide the split when designing the fixture.","notes":"DESIGN NOTE from PR #225 review: the schema allows one blueprint to appear in MULTIPLE things (partial unique index is per-thing only, and test_same_source_allowed_in_different_things pins that behavior). The prod fixture mapping is therefore 1:N — (blueprint_id, thing_id, public_url) derived per thingiverse_files row, not a unique-per-blueprint map. The app's Thingiverse links UI must handle a blueprint linking to several things.","status":"open","priority":2,"issue_type":"feature","owner":"devon.jones@gmail.com","created_at":"2026-07-06T11:55:33.474857151-06:00","created_by":"Devon Jones","updated_at":"2026-07-06T14:51:24.318740515-06:00","dependencies":[{"issue_id":"openforge_catalog-8yb","depends_on_id":"openforge_catalog-750","type":"blocks","created_at":"2026-07-06T11:55:47.512062985-06:00","created_by":"Devon Jones"},{"issue_id":"openforge_catalog-8yb","depends_on_id":"openforge_catalog-4kx","type":"parent-child","created_at":"2026-07-06T11:55:47.882392741-06:00","created_by":"Devon Jones"}]} {"id":"openforge_catalog-ad5","title":"Sync engine: diff local files vs remote hashes, reconcile","description":"For each managed thing: diff manifest-directed local files against remote files using hash comparison (via thingiverse_file ledger) -\u003e upload new, replace changed, delete removed, patch drifted metadata. Idempotent; dry-run mode showing planned actions before mutating. Depends on hash-algorithm verification: if remote is SHA-256, compute+store SHA-256 alongside catalog MD5s.","notes":"PRIOR ART (old v1 tool): tv_update_description did bulk find/replace of boilerplate across all things matching a name filter, with --dry-run (desc_update.json shows real mass migrations happened). tv_tag_dl mirrored all things to local JSON with 3s sleeps (self-rate-limiting; v1 had practical rate limits). Lessons: (1) metadata drift detection should cover the shared-boilerplate case so a boilerplate change re-syncs every managed thing's description; (2) throttle API calls; (3) dry-run is a first-class requirement, already in scope.","status":"open","priority":1,"issue_type":"feature","owner":"devon.jones@gmail.com","created_at":"2026-07-06T11:42:26.401028262-06:00","created_by":"Devon Jones","updated_at":"2026-07-06T12:00:34.191543122-06:00","dependencies":[{"issue_id":"openforge_catalog-ad5","depends_on_id":"openforge_catalog-gn2","type":"blocks","created_at":"2026-07-06T11:42:43.166238543-06:00","created_by":"Devon Jones"},{"issue_id":"openforge_catalog-ad5","depends_on_id":"openforge_catalog-750","type":"blocks","created_at":"2026-07-06T11:42:43.606420026-06:00","created_by":"Devon Jones"},{"issue_id":"openforge_catalog-ad5","depends_on_id":"openforge_catalog-d7a","type":"blocks","created_at":"2026-07-06T11:42:44.00974978-06:00","created_by":"Devon Jones"},{"issue_id":"openforge_catalog-ad5","depends_on_id":"openforge_catalog-4kx","type":"parent-child","created_at":"2026-07-06T11:42:53.860069002-06:00","created_by":"Devon Jones"}]} {"id":"openforge_catalog-bll","title":"Multi-Angle Thumbnail Viewing System","description":"Enable users to view STL models from 10 isometric camera angles with interactive controls. Admins can set default viewing angles per model. System generates 10 isometric-perspective angles (6 primary directions + 4 diagonal corners) during file scan using stl-thumb --cam-pos, combined into single sprite sheet PNG. Frontend has drag-to-rotate and keyboard navigation. All angles maintain 3/4 view for depth, not flat orthographic.","status":"open","priority":1,"issue_type":"epic","created_at":"2025-12-21T09:57:50.607001989-07:00","updated_at":"2025-12-21T10:09:20.457266019-07:00"} {"id":"openforge_catalog-d7a","title":"Manifest schema + metadata assembler","description":"Define per-thing manifest format (YAML): explicit file list for the thing plus field overrides. Assembler builds the thing payload with defaults pulled from catalog Postgres blueprint records (name, description, tag hierarchy -\u003e Thingiverse tags, category, license, images) and applies manifest overrides on top. Zero-manual-entry defaults, curated where it matters. Payload field names/shape finalized against the HAR contract.","notes":"PRIOR ART (old v1 tool): bin/new.*.json templates were per-product-line defaults (openforge2, openlock, construction_kit, miniature, tools): license, category, tags, boilerplate description. Key design lesson: descriptions = per-thing content + SHARED BOILERPLATE FOOTER (Patreon/tutorials/catalog links) that changes over time. Model boilerplate as a shared block composed at assembly time (stored once, referenced by manifests), not baked into each manifest — enables bulk boilerplate updates via re-sync. Old templates: ~/Dropbox/projects/Hardware/objects/OpenForgeDevelopment/openforge/bin/new.*.json","status":"open","priority":2,"issue_type":"feature","owner":"devon.jones@gmail.com","created_at":"2026-07-06T11:42:25.496132115-06:00","created_by":"Devon Jones","updated_at":"2026-07-06T12:00:33.768875881-06:00","dependencies":[{"issue_id":"openforge_catalog-d7a","depends_on_id":"openforge_catalog-4kx","type":"parent-child","created_at":"2026-07-06T11:42:52.892986073-06:00","created_by":"Devon Jones"}]} diff --git a/.beads/last-touched b/.beads/last-touched index bc266d0..2edb454 100644 --- a/.beads/last-touched +++ b/.beads/last-touched @@ -1 +1 @@ -openforge_catalog-7dg +openforge_catalog-8yb diff --git a/openforge/db/schema/version_17.py b/openforge/db/schema/version_17.py new file mode 100644 index 0000000..00cedbd --- /dev/null +++ b/openforge/db/schema/version_17.py @@ -0,0 +1,214 @@ +"""Version 17: Thingiverse sync-state tables + +- Create thingiverse_thing_status enum (draft, published, deleted) +- Create thingiverse_file_type enum (model, image, zip, other) +- Create thingiverse_things table: one row per Thingiverse thing the + publish/sync tool manages (thing_id, status, published/synced + timestamps, remote_metadata_hash for thing-level metadata drift + detection — hash of the last-pushed assembled metadata payload) +- Create thingiverse_files table: the per-file diff ledger tying a thing's + remote files to their catalog sources (blueprints/images/ad-hoc paths) + with local and remote hashes for change detection, plus local_size and + local_mtime as the fallback change signal for file types the remote + API may not hash +- Add indexes on the foreign keys used by sync queries, plus partial + unique indexes preventing duplicate ledger rows for the same local + source within one thing + +Deployment note: the migration creates these tables everywhere it runs +(including prod, where they'll sit empty or mapping-only). The DATA +split is what's local-only: the publish/sync tool populates the full +hash ledger in the local catalog DB, and production receives only the +derived mapping (blueprint_id, thing_id, public_url) via the fixture +path (openforge_catalog-8yb). + +Hash columns: local_md5 mirrors the catalog's existing MD5 tracking; +local_sha256 is included defensively because Thingiverse's remote hash +algorithm is unconfirmed until the API contract work lands +(openforge_catalog-hnr, suspected SHA-256). +""" + +from psycopg import cursor, sql + +from openforge.db.schema import SchemaBase, SchemaVersionDecorator + + +@SchemaVersionDecorator(17) +class SchemaVersion17(SchemaBase): + def up_impl(self, curs: cursor): + self.create_thingiverse_thing_status(curs) + self.create_thingiverse_file_type(curs) + self.create_thingiverse_things(curs) + self.create_thingiverse_files(curs) + self.create_thingiverse_indexes(curs) + + def down_impl(self, curs: cursor): + self.drop_thingiverse_indexes(curs) + self.drop_thingiverse_files(curs) + self.drop_thingiverse_things(curs) + self.drop_thingiverse_file_type(curs) + self.drop_thingiverse_thing_status(curs) + + def create_thingiverse_thing_status(self, curs: cursor): + query = sql.SQL( + """ +CREATE TYPE thingiverse_thing_status + AS ENUM ('draft', 'published', 'deleted') +""" + ) + curs.execute(query) + print(" created thingiverse_thing_status") + + def drop_thingiverse_thing_status(self, curs: cursor): + query = sql.SQL("DROP TYPE thingiverse_thing_status") + curs.execute(query) + print(" dropped thingiverse_thing_status") + + def create_thingiverse_file_type(self, curs: cursor): + query = sql.SQL( + """ +CREATE TYPE thingiverse_file_type + AS ENUM ('model', 'image', 'zip', 'other') +""" + ) + curs.execute(query) + print(" created thingiverse_file_type") + + def drop_thingiverse_file_type(self, curs: cursor): + query = sql.SQL("DROP TYPE thingiverse_file_type") + curs.execute(query) + print(" dropped thingiverse_file_type") + + def create_thingiverse_things(self, curs: cursor): + query = sql.SQL( + """ +CREATE TABLE thingiverse_things ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + thing_id BIGINT, + name TEXT NOT NULL, + public_url TEXT, + manifest_path TEXT, + status thingiverse_thing_status NOT NULL DEFAULT 'draft', + remote_metadata_hash TEXT, + published_at TIMESTAMP, + last_synced_at TIMESTAMP, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE (thing_id) +) +""" + ) + curs.execute(query) + print(" created thingiverse_things") + + def drop_thingiverse_things(self, curs: cursor): + query = sql.SQL("DROP TABLE thingiverse_things") + curs.execute(query) + print(" dropped thingiverse_things") + + def create_thingiverse_files(self, curs: cursor): + # Each row ties one remote file to its local source. Exactly which + # local reference is set depends on file_type: catalog models point + # at blueprints, gallery photos at images, and ad-hoc artifacts + # (zips etc.) carry a filesystem path. At least one must be set. + # Blueprint/image FKs are RESTRICT (not SET NULL): SET NULL would + # trip the CHECK below and make the parent delete fail confusingly; + # RESTRICT makes the rule explicit — unlink the ledger row (so the + # sync engine deletes the remote file) before removing the source. + query = sql.SQL( + """ +CREATE TABLE thingiverse_files ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + thingiverse_thing_id UUID NOT NULL + REFERENCES thingiverse_things(id) ON DELETE CASCADE, + file_type thingiverse_file_type NOT NULL, + blueprint_id UUID REFERENCES blueprints(id) ON DELETE RESTRICT, + image_id UUID REFERENCES images(id) ON DELETE RESTRICT, + local_path TEXT, + local_md5 TEXT, + local_sha256 TEXT, + local_size BIGINT, + local_mtime TIMESTAMP, + remote_file_id BIGINT, + remote_file_name TEXT, + remote_hash TEXT, + last_synced_at TIMESTAMP, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + CHECK (num_nonnulls(blueprint_id, image_id, local_path) >= 1) +) +""" + ) + curs.execute(query) + print(" created thingiverse_files") + + def drop_thingiverse_files(self, curs: cursor): + query = sql.SQL("DROP TABLE thingiverse_files") + curs.execute(query) + print(" dropped thingiverse_files") + + def create_thingiverse_indexes(self, curs: cursor): + query = sql.SQL( + """ +CREATE INDEX idx_thingiverse_files_thing + ON thingiverse_files(thingiverse_thing_id) +""" + ) + curs.execute(query) + query = sql.SQL( + """ +CREATE INDEX idx_thingiverse_files_blueprint + ON thingiverse_files(blueprint_id) +""" + ) + curs.execute(query) + query = sql.SQL( + """ +CREATE INDEX idx_thingiverse_files_image + ON thingiverse_files(image_id) +""" + ) + curs.execute(query) + # Partial unique indexes: the same local source can't appear twice + # in one thing's ledger (each source column independently, since + # only one is set per row). + query = sql.SQL( + """ +CREATE UNIQUE INDEX uniq_thingiverse_files_thing_blueprint + ON thingiverse_files(thingiverse_thing_id, blueprint_id) + WHERE blueprint_id IS NOT NULL +""" + ) + curs.execute(query) + query = sql.SQL( + """ +CREATE UNIQUE INDEX uniq_thingiverse_files_thing_image + ON thingiverse_files(thingiverse_thing_id, image_id) + WHERE image_id IS NOT NULL +""" + ) + curs.execute(query) + query = sql.SQL( + """ +CREATE UNIQUE INDEX uniq_thingiverse_files_thing_path + ON thingiverse_files(thingiverse_thing_id, local_path) + WHERE local_path IS NOT NULL +""" + ) + curs.execute(query) + print(" created thingiverse indexes") + + def drop_thingiverse_indexes(self, curs: cursor): + query = sql.SQL("DROP INDEX uniq_thingiverse_files_thing_path") + curs.execute(query) + query = sql.SQL("DROP INDEX uniq_thingiverse_files_thing_image") + curs.execute(query) + query = sql.SQL("DROP INDEX uniq_thingiverse_files_thing_blueprint") + curs.execute(query) + query = sql.SQL("DROP INDEX idx_thingiverse_files_image") + curs.execute(query) + query = sql.SQL("DROP INDEX idx_thingiverse_files_blueprint") + curs.execute(query) + query = sql.SQL("DROP INDEX idx_thingiverse_files_thing") + curs.execute(query) + print(" dropped thingiverse indexes") diff --git a/tests/conftest.py b/tests/conftest.py index 2beb899..b1595da 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -72,7 +72,8 @@ def clean_tables(test_db): sql.SQL( "TRUNCATE blueprints, tags, images, blueprint_images, " "blueprint_documentation, tag_descriptions, openscad_source, " - "tag_documentation, sessions CASCADE" + "tag_documentation, sessions, thingiverse_things, " + "thingiverse_files CASCADE" ) ) conn.commit() diff --git a/tests/test_thingiverse_schema.py b/tests/test_thingiverse_schema.py new file mode 100644 index 0000000..ee150f6 --- /dev/null +++ b/tests/test_thingiverse_schema.py @@ -0,0 +1,237 @@ +"""Schema-semantics tests for the thingiverse sync-state tables (version 17).""" + +import psycopg +import pytest +from psycopg import sql +from psycopg.rows import dict_row + +import openforge.db.sql.blueprints as blueprint_sql +import openforge.db.sql.images as image_sql + +from .test_helpers import create_test_blueprint + + +def insert_thing(curs, name="Test Thing", thing_id=None, status="draft"): + curs.execute( + sql.SQL( + "INSERT INTO thingiverse_things (name, thing_id, status) " + "VALUES (%s, %s, %s) RETURNING *" + ), + (name, thing_id, status), + ) + return curs.fetchone() + + +def insert_file(curs, thing_row_id, **overrides): + fields = { + "thingiverse_thing_id": thing_row_id, + "file_type": "model", + "blueprint_id": None, + "image_id": None, + "local_path": None, + "local_md5": None, + "local_sha256": None, + "remote_file_id": None, + "remote_file_name": None, + "remote_hash": None, + } + fields.update(overrides) + columns = list(fields.keys()) + curs.execute( + sql.SQL("INSERT INTO thingiverse_files ({}) VALUES ({}) RETURNING *").format( + sql.SQL(", ").join(sql.Identifier(c) for c in columns), + sql.SQL(", ").join(sql.Placeholder() for _ in columns), + ), + list(fields.values()), + ) + return curs.fetchone() + + +def test_thing_round_trip(test_db): + with test_db.connection() as conn: + with conn.cursor(row_factory=dict_row) as curs: + thing = insert_thing(curs, name="Cut Stone Walls", thing_id=12345) + assert thing["status"] == "draft" + assert thing["thing_id"] == 12345 + assert thing["created_at"] is not None + + +def test_thing_id_unique_but_nullable(test_db): + with test_db.connection() as conn: + with conn.cursor(row_factory=dict_row) as curs: + # drafts have no remote id yet; multiple NULLs must coexist + insert_thing(curs, name="Draft A") + insert_thing(curs, name="Draft B") + insert_thing(curs, name="Published", thing_id=99) + with pytest.raises(psycopg.errors.UniqueViolation): + insert_thing(curs, name="Duplicate", thing_id=99) + + +def test_file_requires_a_local_source(test_db): + with test_db.connection() as conn: + with conn.cursor(row_factory=dict_row) as curs: + thing = insert_thing(curs) + with pytest.raises(psycopg.errors.CheckViolation): + insert_file(curs, thing["id"]) # no blueprint/image/path + + +def test_file_accepts_each_local_source_kind(test_db): + with test_db.connection() as conn: + with conn.cursor(row_factory=dict_row) as curs: + blueprint = blueprint_sql.insert_blueprint(curs, create_test_blueprint()) + thing = insert_thing(curs) + + by_blueprint = insert_file( + curs, thing["id"], blueprint_id=blueprint["id"], local_md5="abc123" + ) + assert by_blueprint["file_type"] == "model" + + by_path = insert_file( + curs, thing["id"], file_type="zip", local_path="/out/set.zip" + ) + assert by_path["local_path"] == "/out/set.zip" + + image = image_sql.insert_image( + curs, "gallery_shot", "http://test.com/shot.jpg" + ) + by_image = insert_file( + curs, thing["id"], file_type="image", image_id=image["id"] + ) + assert by_image["image_id"] == image["id"] + + +def test_deleting_thing_cascades_to_files(test_db): + with test_db.connection() as conn: + with conn.cursor(row_factory=dict_row) as curs: + thing = insert_thing(curs) + insert_file(curs, thing["id"], local_path="/out/a.stl") + curs.execute( + sql.SQL("DELETE FROM thingiverse_things WHERE id = %s"), + (thing["id"],), + ) + curs.execute(sql.SQL("SELECT count(*) AS n FROM thingiverse_files")) + assert curs.fetchone()["n"] == 0 + + +def test_deleting_referenced_blueprint_is_restricted(test_db): + with test_db.connection() as conn: + with conn.cursor(row_factory=dict_row) as curs: + blueprint = blueprint_sql.insert_blueprint(curs, create_test_blueprint()) + thing = insert_thing(curs) + insert_file(curs, thing["id"], blueprint_id=blueprint["id"]) + # RESTRICT raises RestrictViolation (23001) on newer Postgres, + # ForeignKeyViolation (23503) on older — accept either + with pytest.raises( + ( + psycopg.errors.ForeignKeyViolation, + psycopg.errors.RestrictViolation, + ) + ): + curs.execute( + sql.SQL("DELETE FROM blueprints WHERE id = %s"), + (blueprint["id"],), + ) + + +def test_deleting_referenced_image_is_restricted(test_db): + with test_db.connection() as conn: + with conn.cursor(row_factory=dict_row) as curs: + image = image_sql.insert_image( + curs, "gallery_shot", "http://test.com/shot.jpg" + ) + thing = insert_thing(curs) + insert_file(curs, thing["id"], file_type="image", image_id=image["id"]) + with pytest.raises( + ( + psycopg.errors.ForeignKeyViolation, + psycopg.errors.RestrictViolation, + ) + ): + curs.execute( + sql.SQL("DELETE FROM images WHERE id = %s"), (image["id"],) + ) + + +def test_thing_id_unique_on_update(test_db): + with test_db.connection() as conn: + with conn.cursor(row_factory=dict_row) as curs: + insert_thing(curs, name="Existing", thing_id=99) + draft = insert_thing(curs, name="Draft") + with pytest.raises(psycopg.errors.UniqueViolation): + curs.execute( + sql.SQL( + "UPDATE thingiverse_things SET thing_id = 99 WHERE id = %s" + ), + (draft["id"],), + ) + + +def test_file_check_enforced_on_update(test_db): + with test_db.connection() as conn: + with conn.cursor(row_factory=dict_row) as curs: + thing = insert_thing(curs) + row = insert_file(curs, thing["id"], local_path="/out/a.stl") + with pytest.raises(psycopg.errors.CheckViolation): + curs.execute( + sql.SQL( + "UPDATE thingiverse_files SET local_path = NULL WHERE id = %s" + ), + (row["id"],), + ) + + +def test_duplicate_source_in_same_thing_rejected(test_db): + with test_db.connection() as conn: + with conn.cursor(row_factory=dict_row) as curs: + blueprint = blueprint_sql.insert_blueprint(curs, create_test_blueprint()) + thing = insert_thing(curs) + insert_file(curs, thing["id"], blueprint_id=blueprint["id"]) + with pytest.raises(psycopg.errors.UniqueViolation): + insert_file(curs, thing["id"], blueprint_id=blueprint["id"]) + + +def test_duplicate_image_in_same_thing_rejected(test_db): + with test_db.connection() as conn: + with conn.cursor(row_factory=dict_row) as curs: + image = image_sql.insert_image( + curs, "gallery_shot", "http://test.com/shot.jpg" + ) + thing = insert_thing(curs) + insert_file(curs, thing["id"], file_type="image", image_id=image["id"]) + with pytest.raises(psycopg.errors.UniqueViolation): + insert_file(curs, thing["id"], file_type="image", image_id=image["id"]) + + +def test_duplicate_path_in_same_thing_rejected(test_db): + with test_db.connection() as conn: + with conn.cursor(row_factory=dict_row) as curs: + thing = insert_thing(curs) + insert_file(curs, thing["id"], file_type="zip", local_path="/out/a.zip") + with pytest.raises(psycopg.errors.UniqueViolation): + insert_file(curs, thing["id"], file_type="zip", local_path="/out/a.zip") + + +def test_same_source_allowed_in_different_things(test_db): + with test_db.connection() as conn: + with conn.cursor(row_factory=dict_row) as curs: + blueprint = blueprint_sql.insert_blueprint(curs, create_test_blueprint()) + thing_a = insert_thing(curs, name="Set A") + thing_b = insert_thing(curs, name="Set B") + insert_file(curs, thing_a["id"], blueprint_id=blueprint["id"]) + # a blueprint may legitimately ship in more than one thing + insert_file(curs, thing_b["id"], blueprint_id=blueprint["id"]) + + +def test_file_type_enum_rejects_unknown_values(test_db): + with test_db.connection() as conn: + with conn.cursor(row_factory=dict_row) as curs: + thing = insert_thing(curs) + with pytest.raises(psycopg.errors.InvalidTextRepresentation): + insert_file(curs, thing["id"], file_type="hologram", local_path="/x") + + +def test_thing_status_enum_rejects_unknown_values(test_db): + with test_db.connection() as conn: + with conn.cursor(row_factory=dict_row) as curs: + with pytest.raises(psycopg.errors.InvalidTextRepresentation): + insert_thing(curs, status="imaginary")