From a2c2985a623daeeccecc6303787e9e8f52560630 Mon Sep 17 00:00:00 2001 From: Frida Rosenaa Date: Sat, 10 Jan 2026 13:18:12 +0100 Subject: [PATCH 01/12] Fix webui id gen bug --- webui/src/lib/globalHelpers.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/webui/src/lib/globalHelpers.ts b/webui/src/lib/globalHelpers.ts index 0a7db20c81..00f3c36293 100644 --- a/webui/src/lib/globalHelpers.ts +++ b/webui/src/lib/globalHelpers.ts @@ -93,7 +93,9 @@ export const getIdFromName = (name: string): string => { name .trim() .toLowerCase() - .replace(/\s+/g, '_'), + .replace(/\s+/g, '_') + .replace(' ', '_') + .replace('-', '_'), ['+'] ); }; From 4895125e554352f8bd2b0ca67221a1d1d9d0affa Mon Sep 17 00:00:00 2001 From: Frida Rosenaa Date: Sat, 10 Jan 2026 13:20:56 +0100 Subject: [PATCH 02/12] Make update profliles not run on PR's --- .github/workflows/update_profiles.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/update_profiles.yaml b/.github/workflows/update_profiles.yaml index 0213e310ed..ae26aa7284 100644 --- a/.github/workflows/update_profiles.yaml +++ b/.github/workflows/update_profiles.yaml @@ -4,9 +4,14 @@ on: schedule: - cron: "0 0 * * *" workflow_dispatch: + push: + branches: + - main jobs: update_slicer_profiles: + # Only run on main branch, not on PRs + if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request' runs-on: ubuntu-latest permissions: contents: write From df41169bcc73dd1e953013493d8b0713e543059e Mon Sep 17 00:00:00 2001 From: Frida Rosenaa Date: Sat, 10 Jan 2026 14:18:09 +0100 Subject: [PATCH 03/12] Remove temporary script from schema migration --- scripts/migrate_schema.py | 1229 ------------------------------------- 1 file changed, 1229 deletions(-) delete mode 100644 scripts/migrate_schema.py diff --git a/scripts/migrate_schema.py b/scripts/migrate_schema.py deleted file mode 100644 index dc9ef208ed..0000000000 --- a/scripts/migrate_schema.py +++ /dev/null @@ -1,1229 +0,0 @@ -#!/usr/bin/env python3 -""" -migrate_schema.py - Migrate data files to new schema format - -This script transforms the existing data files to conform to the updated schemas: -- brand.json: Rename 'brand' -> 'name', add 'id', rename logo files, rename folder -- filament.json: Add 'id' field, rename folder -- variant.json: Rename 'color_name' -> 'name', add 'id', rename folder -- store.json: Rename logo files, rename folder -- sizes.json: Update store_id references, migrate spool_refill to size level -- material.json: Map material types to valid enum values from material_types_schema.json - -The script processes stores first to discover ID mappings (old_id -> new_id), -then uses these mappings to automatically update store_id references in data files. - -For spool_refill migration: The new schema moves spool_refill from purchase_link -level to size level. If a size has mixed refill/non-refill purchase links, they -are split into separate size entries. - -For material merging: When a material type maps to an existing folder (e.g., -"Carbon Fibre" -> "PETG"), filaments are merged. Unique filaments are moved, -duplicate filaments have their data merged (missing variants/fields copied), -then the source material folder is deleted. - -Usage: - python scripts/migrate_schema.py --dry-run # Preview changes - python scripts/migrate_schema.py # Apply changes -""" - -import argparse -import json -import os -import re -import shutil -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - - -# Mapping from old material names to schema enum values -# See schemas/material_types_schema.json for valid enum values -MATERIAL_TYPE_MAP: dict[str, str] = { - # Nylon variants -> PA types - "Nylon": "PA12", - "PAHT": "PA12", - "PA": "PA6", - "PA-CF": "PA6", - "PA-GF": "PA6", - # PETG variants - "PETG-CF": "PETG", - "Addnite": "PETG", - "Carbon Fibre": "PETG", # Add-North Carbon Fibre is PETG-based - "Graphene": "PETG", # Add-North Graphene is PETG-based - # PET variants - "PE": "PET", - "PET-CF": "PET", - # PLA variants - "APLA": "PLA", - "Bio-performance": "PLA", - "AquaPrint": "PLA", - "Woodfill": "PLA", - "Special": "PLA", - # PPA variants - "PPA-CF": "PPA", - # CPE variants - "CPE-HT": "CPE", - # PP variants - "CPX-PP": "PP", - # PC variants - "CPX-KyronMax": "PC", - # EVA variants - "EVA+": "EVA", - # Support materials (typically PVA-based) - "Support": "PVA", - # Composite materials - "Boron Carbide": "PA12", -} - - -@dataclass -class MigrationIssue: - """Represents an issue encountered during migration.""" - issue_type: str - path: str - message: str - - -@dataclass -class MigrationReport: - """Collects issues and warnings during migration.""" - issues: list[MigrationIssue] = field(default_factory=list) - - def add_issue(self, issue_type: str, path: str, message: str) -> None: - self.issues.append(MigrationIssue(issue_type, path, message)) - - def has_issues(self) -> bool: - return len(self.issues) > 0 - - def print_summary(self) -> None: - if not self.issues: - print("\nNo issues encountered during migration.") - return - - print(f"\n{'=' * 60}") - print(f"MIGRATION ISSUES ({len(self.issues)} total)") - print('=' * 60) - - # Group by issue type - by_type: dict[str, list[MigrationIssue]] = {} - for issue in self.issues: - by_type.setdefault(issue.issue_type, []).append(issue) - - for issue_type, issues in sorted(by_type.items()): - print(f"\n{issue_type} ({len(issues)}):") - for issue in issues: - print(f" - {issue.path}") - print(f" {issue.message}") - - -def generate_id(name: str) -> str: - """Convert a name to a valid id (lowercase, spaces/hyphens/ampersands to underscores). - - Examples: - "Bambu Lab" -> "bambu_lab" - "3DO" -> "3do" - "Add-North" -> "add_north" - "ABS+" -> "abs+" - "Black&White" -> "black_white" - """ - id_str = name.lower() - id_str = re.sub(r'[\s\-&]+', '_', id_str) # spaces/hyphens/ampersands -> underscore - id_str = re.sub(r'[^a-z0-9_+]', '', id_str) # remove invalid chars (keep +) - id_str = re.sub(r'_+', '_', id_str) # collapse multiple underscores - return id_str.strip('_') - - -def rename_folder(folder: Path, new_name: str, dry_run: bool) -> Path: - """Rename a folder to a new name. - - Handles case-only renames on case-insensitive filesystems by using - a two-step rename through a temporary name. - - Returns: - The new path after renaming (or the original if no rename needed). - """ - if folder.name == new_name: - return folder - - new_path = folder.parent / new_name - if new_path.exists(): - # Check if it's the same folder (case-insensitive rename like Black-White -> black_white) - try: - if folder.samefile(new_path): - # Same folder, different case - do a two-step rename via temp name - if dry_run: - print(f" Would rename folder: {folder.name} -> {new_name}") - else: - temp_path = folder.parent / f".{new_name}_temp_{os.getpid()}" - shutil.move(str(folder), str(temp_path)) - shutil.move(str(temp_path), str(new_path)) - print(f" Renamed folder: {folder.name} -> {new_name}") - return new_path - except (OSError, ValueError): - pass - - print(f" Warning: Cannot rename {folder.name} -> {new_name}, target exists") - return folder - - if dry_run: - print(f" Would rename folder: {folder.name} -> {new_name}") - else: - shutil.move(str(folder), str(new_path)) - print(f" Renamed folder: {folder.name} -> {new_name}") - - return new_path - - -def find_logo_file(directory: Path) -> Path | None: - """Find a logo file in the given directory.""" - logo_extensions = ['.png', '.jpg', '.jpeg', '.svg'] - for f in directory.iterdir(): - if f.is_file() and f.suffix.lower() in logo_extensions: - return f - return None - - -def rename_logo(directory: Path, current_logo: str, dry_run: bool) -> str | None: - """Rename logo file to logo.{ext} and return new name. - - Returns: - New logo filename (e.g., "logo.png") or None if no logo found. - """ - if not current_logo: - return None - - # Already in correct format - if re.match(r'^logo\.(png|jpg|svg)$', current_logo): - return current_logo - - # Find the current logo file - current_path = directory / current_logo - if not current_path.exists(): - # Try to find any logo file - logo_file = find_logo_file(directory) - if logo_file: - current_path = logo_file - else: - print(f" Warning: Logo file not found: {current_logo} in {directory}") - return None - - # Determine new name - ext = current_path.suffix.lower() - if ext == '.jpeg': - ext = '.jpg' - new_name = f"logo{ext}" - new_path = directory / new_name - - if current_path != new_path: - if dry_run: - print(f" Would rename logo: {current_path.name} -> {new_name}") - else: - shutil.move(str(current_path), str(new_path)) - print(f" Renamed logo: {current_path.name} -> {new_name}") - - return new_name - - -def load_json(path: Path) -> dict[str, Any] | None: - """Load a JSON file, returning None if it doesn't exist.""" - if not path.exists(): - return None - with open(path, 'r', encoding='utf-8') as f: - return json.load(f) - - -def save_json(path: Path, data: dict[str, Any], dry_run: bool) -> None: - """Save data to a JSON file.""" - if dry_run: - return - with open(path, 'w', encoding='utf-8') as f: - json.dump(data, f, indent=4, ensure_ascii=False) - f.write('\n') - - -def migrate_spool_refill(sizes_data: list[dict]) -> tuple[list[dict], bool]: - """Migrate spool_refill from purchase_link level to size level. - - Per the new schema, spool_refill is now on the size object and applies to all - purchase_links within that size. If a size has mixed refill/non-refill links, - we split them into separate size entries. - - Args: - sizes_data: List of size objects from sizes.json. - - Returns: - Tuple of (updated sizes list, whether changes were made). - """ - new_sizes = [] - changed = False - - for size_obj in sizes_data: - if not isinstance(size_obj, dict): - new_sizes.append(size_obj) - continue - - purchase_links = size_obj.get('purchase_links', []) - if not purchase_links: - new_sizes.append(size_obj) - continue - - # Separate links by spool_refill status - refill_links = [] - non_refill_links = [] - - for link in purchase_links: - if not isinstance(link, dict): - non_refill_links.append(link) - continue - - is_refill = link.get('spool_refill', False) - # Remove spool_refill from the link (it's deprecated at this level) - link_copy = {k: v for k, v in link.items() if k != 'spool_refill'} - - if is_refill: - refill_links.append(link_copy) - changed = True # We're removing/moving spool_refill - else: - # Only mark changed if we actually removed a spool_refill key - if 'spool_refill' in link: - changed = True - non_refill_links.append(link_copy) - - # Create size entries based on what we found - if non_refill_links: - # Non-refill size (original size without spool_refill) - non_refill_size = {k: v for k, v in size_obj.items() if k != 'purchase_links'} - non_refill_size['purchase_links'] = non_refill_links - # Ensure spool_refill is not set or is False - non_refill_size.pop('spool_refill', None) - new_sizes.append(non_refill_size) - - if refill_links: - # Refill size (with spool_refill=true at size level) - refill_size = {k: v for k, v in size_obj.items() if k != 'purchase_links'} - refill_size['spool_refill'] = True - refill_size['purchase_links'] = refill_links - new_sizes.append(refill_size) - changed = True - - return new_sizes, changed - - -def migrate_sizes( - sizes_file: Path, - id_mapping: dict[str, str], - valid_store_ids: set[str], - dry_run: bool -) -> bool: - """Update store_id references and migrate spool_refill in a sizes.json file. - - Args: - sizes_file: Path to the sizes.json file. - id_mapping: Dictionary mapping old store IDs to new store IDs. - valid_store_ids: Set of valid store IDs from stores directory. - dry_run: If True, don't actually modify files. - - Returns: - True if changes were made, False otherwise. - """ - data = load_json(sizes_file) - if data is None: - return False - - file_changed = False - - # sizes.json is a list of size objects - if isinstance(data, list): - # First, migrate spool_refill from purchase_link level to size level - data, spool_refill_changed = migrate_spool_refill(data) - if spool_refill_changed: - print(f" Migrated spool_refill to size level") - file_changed = True - - # Then update store_id references - for size_obj in data: - if not isinstance(size_obj, dict): - continue - purchase_links = size_obj.get('purchase_links', []) - if not isinstance(purchase_links, list): - continue - - for link in purchase_links: - if not isinstance(link, dict): - continue - old_store_id = link.get('store_id') - if not old_store_id: - continue - - # Find the matching store ID - new_store_id = find_matching_store_id( - old_store_id, id_mapping, valid_store_ids - ) - - if new_store_id is None: - # No match found - will be reported by caller - continue - - if old_store_id != new_store_id: - link['store_id'] = new_store_id - print(f" Updated store_id: {old_store_id} -> {new_store_id}") - file_changed = True - - if file_changed: - save_json(sizes_file, data, dry_run) - - return file_changed - - -def merge_variant_dirs( - source_dir: Path, target_dir: Path, dry_run: bool -) -> bool: - """Merge data from source variant directory into target if target is missing data. - - Args: - source_dir: The source variant directory (will be deleted after merge). - target_dir: The target variant directory to merge into. - dry_run: If True, don't actually modify files. - - Returns: - True if merge was successful (source can be deleted), False otherwise. - """ - # Merge variant.json data if target is missing fields - source_variant = load_json(source_dir / 'variant.json') - target_variant = load_json(target_dir / 'variant.json') - - if source_variant and target_variant: - merged = False - for key, value in source_variant.items(): - if key not in target_variant and key != 'id': - target_variant[key] = value - merged = True - if dry_run: - print(f" Would merge field '{key}' into target variant.json") - else: - print(f" Merged field '{key}' into target variant.json") - - if merged and not dry_run: - save_json(target_dir / 'variant.json', target_variant, dry_run) - - # Merge sizes.json if source has data target doesn't - source_sizes = source_dir / 'sizes.json' - target_sizes = target_dir / 'sizes.json' - - if source_sizes.exists(): - source_data = load_json(source_sizes) - target_data = load_json(target_sizes) if target_sizes.exists() else [] - - if source_data and isinstance(source_data, list): - if not target_data: - # Target has no sizes, copy from source - if dry_run: - print(f" Would copy sizes.json to target") - else: - shutil.copy2(str(source_sizes), str(target_sizes)) - print(f" Copied sizes.json to target") - else: - if dry_run: - print(f" Would skip sizes.json (target already has data)") - else: - print(f" Skipped sizes.json (target already has data)") - - return True - - -def migrate_variant( - variant_dir: Path, - dry_run: bool, - id_mapping: dict[str, str] | None = None, - valid_store_ids: set[str] | None = None -) -> tuple[bool, Path]: - """Transform variant.json: color_name->name, add id, rename folder. - Also updates store_id references in sizes.json. - - If a folder with the target name already exists (e.g., renaming "Black-White" to - "black_white" when "black_white" already exists), merge the contents and delete the source. - - Args: - variant_dir: Path to the variant directory. - dry_run: If True, don't actually modify files. - id_mapping: Dictionary mapping old store IDs to new store IDs. - valid_store_ids: Set of valid store IDs from stores directory. - - Returns: - Tuple of (changes_made, new_path after folder rename). - """ - variant_file = variant_dir / 'variant.json' - data = load_json(variant_file) - if data is None: - return False, variant_dir - - changed = False - folder_name = variant_dir.name - new_id = generate_id(folder_name) - - # Add id if missing - if 'id' not in data: - data['id'] = new_id - print(f" Added id: {data['id']}") - changed = True - - # Rename 'color_name' to 'name' - if 'color_name' in data: - data['name'] = data.pop('color_name') - print(f" Renamed 'color_name' -> 'name': {data['name']}") - changed = True - - if changed: - # Reorder keys: id first, then name, then color_hex, then rest - ordered = {} - for key in ['id', 'name', 'color_hex']: - if key in data: - ordered[key] = data[key] - for key in data: - if key not in ordered: - ordered[key] = data[key] - save_json(variant_file, ordered, dry_run) - - # Update store_id references in sizes.json - sizes_file = variant_dir / 'sizes.json' - if sizes_file.exists() and id_mapping is not None and valid_store_ids is not None: - if migrate_sizes(sizes_file, id_mapping, valid_store_ids, dry_run): - changed = True - - # Check if target folder already exists (duplicate scenario) - target_dir = variant_dir.parent / new_id - if target_dir.exists() and target_dir != variant_dir: - # Target exists - merge into it and delete source - print(f" Merging duplicate variant: {folder_name} -> {new_id}") - merge_variant_dirs(variant_dir, target_dir, dry_run) - if dry_run: - print(f" Would delete merged source: {folder_name}") - else: - shutil.rmtree(str(variant_dir)) - print(f" Deleted merged source: {folder_name}") - return True, target_dir - - # Rename folder to match id - new_dir = rename_folder(variant_dir, new_id, dry_run) - if new_dir != variant_dir: - changed = True - - return changed, new_dir - - -def merge_filament_dirs( - source_dir: Path, target_dir: Path, dry_run: bool -) -> bool: - """Merge data from source filament directory into target if target is missing data. - - Compares the two directories and merges any missing variants or data from source - into target. After merging, the source can be safely deleted. - - Args: - source_dir: The source filament directory (will be deleted after merge). - target_dir: The target filament directory to merge into. - dry_run: If True, don't actually modify files. - - Returns: - True if merge was successful (source can be deleted), False otherwise. - """ - # Get variants from both directories - source_variants = {d.name: d for d in source_dir.iterdir() if d.is_dir()} - target_variants = {d.name: d for d in target_dir.iterdir() if d.is_dir()} - - # Merge missing variants from source to target - for variant_name, variant_dir in source_variants.items(): - if variant_name not in target_variants: - # Variant doesn't exist in target - move it - dest = target_dir / variant_name - if dry_run: - print(f" Would move variant: {variant_name} -> {target_dir.name}/") - else: - shutil.move(str(variant_dir), str(dest)) - print(f" Moved variant: {variant_name} -> {target_dir.name}/") - else: - # Variant exists in both - check if we need to merge sizes.json - source_sizes = variant_dir / 'sizes.json' - target_sizes = target_variants[variant_name] / 'sizes.json' - - if source_sizes.exists(): - source_data = load_json(source_sizes) - target_data = load_json(target_sizes) if target_sizes.exists() else [] - - if source_data and target_data is not None: - # Check for purchase links or sizes that might be missing - # For simplicity, if source has data target doesn't, log it - if dry_run: - print(f" Would skip duplicate variant: {variant_name} (exists in target)") - else: - print(f" Skipped duplicate variant: {variant_name} (exists in target)") - - # Merge filament.json data if target is missing fields - source_filament = load_json(source_dir / 'filament.json') - target_filament = load_json(target_dir / 'filament.json') - - if source_filament and target_filament: - merged = False - for key, value in source_filament.items(): - if key not in target_filament and key != 'id': - target_filament[key] = value - merged = True - if dry_run: - print(f" Would merge field '{key}' into target filament.json") - else: - print(f" Merged field '{key}' into target filament.json") - - if merged and not dry_run: - save_json(target_dir / 'filament.json', target_filament, dry_run) - - return True - - -def migrate_filament(filament_dir: Path, dry_run: bool) -> tuple[bool, Path]: - """Transform filament.json: add id, rename folder. - - If a folder with the target name already exists (e.g., renaming "Rigid X" to - "rigid_x" when "rigid_x" already exists), merge the contents and delete the source. - - Returns: - Tuple of (changes_made, new_path after folder rename). - """ - filament_file = filament_dir / 'filament.json' - data = load_json(filament_file) - if data is None: - return False, filament_dir - - changed = False - folder_name = filament_dir.name - new_id = generate_id(folder_name) - - # Add or update id to match folder - if data.get('id') != new_id: - old_id = data.get('id') - data['id'] = new_id - if old_id: - print(f" Updated id: {old_id} -> {new_id}") - else: - print(f" Added id: {new_id}") - changed = True - - if changed: - # Reorder keys: id first, then name, then rest - ordered = {} - for key in ['id', 'name']: - if key in data: - ordered[key] = data[key] - for key in data: - if key not in ordered: - ordered[key] = data[key] - save_json(filament_file, ordered, dry_run) - - # Check if target folder already exists (duplicate scenario) - target_dir = filament_dir.parent / new_id - if target_dir.exists() and target_dir != filament_dir: - # Target exists - merge into it and delete source - print(f" Merging duplicate: {folder_name} -> {new_id}") - merge_filament_dirs(filament_dir, target_dir, dry_run) - if dry_run: - print(f" Would delete merged source: {folder_name}") - else: - shutil.rmtree(str(filament_dir)) - print(f" Deleted merged source: {folder_name}") - return True, target_dir - - # Rename folder to match id - new_dir = rename_folder(filament_dir, new_id, dry_run) - if new_dir != filament_dir: - changed = True - - return changed, new_dir - - -def migrate_material( - material_dir: Path, dry_run: bool, report: MigrationReport -) -> tuple[bool, Path | None]: - """Transform material.json: map material types to enum values, rename folder. - - Note: material.json only has 'material', 'default_max_dry_temperature', - and 'default_slicer_settings' fields - no 'id' field. - - If the target material folder already exists (e.g., mapping APLA -> PLA when - PLA folder exists), moves all filament subdirectories to the target folder - and deletes the source folder. - - Returns: - Tuple of (changes_made, new_path after folder rename). - If the folder was merged into another and deleted, returns (True, None). - """ - material_file = material_dir / 'material.json' - data = load_json(material_file) - if data is None: - return False, material_dir - - changed = False - folder_name = material_dir.name - original_material = data.get('material', folder_name) - - # Remove 'id' field if present (not part of material schema) - if 'id' in data: - del data['id'] - print(f" Removed invalid 'id' field from material.json") - changed = True - - # Map material type to enum value if needed - new_material = original_material - if original_material in MATERIAL_TYPE_MAP: - new_material = MATERIAL_TYPE_MAP[original_material] - data['material'] = new_material - print(f" Mapped material: {original_material} -> {new_material}") - changed = True - else: - # Check if material is already a valid enum value (all caps typically) - # If not, record it as an unmapped material - material_value = data.get('material', '') - if material_value and not material_value.isupper(): - # Likely needs mapping but we don't have one - report.add_issue( - "Unmapped material type", - str(material_dir.relative_to(material_dir.parent.parent)), - f"Material '{material_value}' is not mapped to a schema enum value" - ) - - # Check if target folder already exists (merge scenario) - new_folder_name = data.get('material', folder_name) - target_dir = material_dir.parent / new_folder_name - - if target_dir.exists() and target_dir != material_dir: - # Target folder exists - merge filaments into it - print(f" Merging into existing folder: {new_folder_name}") - - # Build a mapping of normalized names to actual folder names in target - target_filaments: dict[str, Path] = {} - for d in target_dir.iterdir(): - if d.is_dir(): - # Map both the actual name and the normalized name - target_filaments[d.name] = d - target_filaments[generate_id(d.name)] = d - - # Move all filament subdirectories to target - filament_dirs = [d for d in material_dir.iterdir() if d.is_dir()] - for filament_dir in filament_dirs: - # Check if a matching filament exists in target (by name or normalized name) - source_name = filament_dir.name - source_normalized = generate_id(source_name) - - matching_target = target_filaments.get(source_name) or target_filaments.get(source_normalized) - - if matching_target: - # Filament exists in both - merge data from source to target - print(f" Merging duplicate filament: {source_name} -> {matching_target.name}") - merge_filament_dirs(filament_dir, matching_target, dry_run) - # Delete the source filament directory after merge - if dry_run: - print(f" Would delete merged source: {source_name}") - else: - shutil.rmtree(str(filament_dir)) - print(f" Deleted merged source: {source_name}") - else: - # Filament doesn't exist in target - move it - dest = target_dir / source_name - if dry_run: - print(f" Would move filament: {source_name} -> {new_folder_name}/") - else: - shutil.move(str(filament_dir), str(dest)) - print(f" Moved filament: {source_name} -> {new_folder_name}/") - - # Delete the now-empty source folder (including material.json) - if dry_run: - print(f" Would delete empty folder: {folder_name}") - else: - shutil.rmtree(str(material_dir)) - print(f" Deleted empty folder: {folder_name}") - - return True, None # Signal that folder was merged and deleted - - # Normal case: just update material.json and rename folder - if changed: - # Reorder keys: material first, then rest - ordered = {} - if 'material' in data: - ordered['material'] = data['material'] - for key in data: - if key not in ordered: - ordered[key] = data[key] - save_json(material_file, ordered, dry_run) - - # Rename folder to match material type (keep uppercase, e.g., "PLA", "CF") - new_dir = rename_folder(material_dir, new_folder_name, dry_run) - if new_dir != material_dir: - changed = True - - return changed, new_dir - - -def migrate_brand(brand_dir: Path, dry_run: bool) -> tuple[bool, Path]: - """Transform brand.json: brand->name, add id, rename logo, rename folder. - - Returns: - Tuple of (changes_made, new_path after folder rename). - """ - brand_file = brand_dir / 'brand.json' - data = load_json(brand_file) - if data is None: - return False, brand_dir - - changed = False - folder_name = brand_dir.name - new_id = generate_id(folder_name) - - # Add id if missing - if 'id' not in data: - data['id'] = new_id - print(f" Added id: {data['id']}") - changed = True - - # Rename 'brand' to 'name' - if 'brand' in data: - data['name'] = data.pop('brand') - print(f" Renamed 'brand' -> 'name': {data['name']}") - changed = True - - # Rename logo file and update reference - if 'logo' in data: - new_logo = rename_logo(brand_dir, data['logo'], dry_run) - if new_logo and new_logo != data['logo']: - data['logo'] = new_logo - changed = True - - if changed: - # Reorder keys: id first, then name, then rest - ordered = {} - for key in ['id', 'name', 'website', 'logo', 'origin']: - if key in data: - ordered[key] = data[key] - for key in data: - if key not in ordered: - ordered[key] = data[key] - save_json(brand_file, ordered, dry_run) - - # Rename folder to match id - new_dir = rename_folder(brand_dir, new_id, dry_run) - if new_dir != brand_dir: - changed = True - - return changed, new_dir - - -def migrate_store( - store_dir: Path, dry_run: bool, id_mapping: dict[str, str] -) -> tuple[bool, Path]: - """Transform store.json: rename logo file, rename folder. - - Args: - store_dir: Path to the store directory. - dry_run: If True, don't actually modify files. - id_mapping: Dictionary to populate with old_id -> new_id mappings. - - Returns: - Tuple of (changes_made, new_path after folder rename). - """ - store_file = store_dir / 'store.json' - data = load_json(store_file) - if data is None: - return False, store_dir - - changed = False - folder_name = store_dir.name - new_id = generate_id(folder_name) - - # Record the old ID -> new ID mapping - old_id = data.get('id', folder_name) - if old_id != new_id: - id_mapping[old_id] = new_id - # Also map the folder name if different from the old id - if folder_name != old_id and folder_name != new_id: - id_mapping[folder_name] = new_id - - # Update id in data if it doesn't match the new format - if 'id' in data and data['id'] != new_id: - data['id'] = new_id - print(f" Updated id: {data['id']}") - changed = True - - # Rename logo file and update reference - if 'logo' in data: - new_logo = rename_logo(store_dir, data['logo'], dry_run) - if new_logo and new_logo != data['logo']: - data['logo'] = new_logo - changed = True - - if changed: - save_json(store_file, data, dry_run) - - # Rename folder to match id - new_dir = rename_folder(store_dir, new_id, dry_run) - if new_dir != store_dir: - changed = True - - return changed, new_dir - - -def process_data_directory( - data_dir: Path, - dry_run: bool, - report: MigrationReport, - id_mapping: dict[str, str] | None = None, - valid_store_ids: set[str] | None = None -) -> dict[str, int]: - """Process all data files in the data directory. - - Important: We process from deepest level up (variants -> filaments -> materials -> brands) - to avoid path issues when renaming folders. - - Args: - data_dir: Path to the data directory. - dry_run: If True, don't actually modify files. - report: MigrationReport to add issues to. - id_mapping: Dictionary mapping old store IDs to new store IDs. - valid_store_ids: Set of valid store IDs from stores directory. - - Returns: - Dictionary with counts of modified files by type. - """ - stats = {'brands': 0, 'materials': 0, 'filaments': 0, 'variants': 0} - - # Collect all directories first to avoid issues with iteration during rename - brand_dirs = sorted([d for d in data_dir.iterdir() if d.is_dir()]) - - for brand_dir in brand_dirs: - print(f"\nBrand: {brand_dir.name}") - - # First, process all nested content (variants -> filaments -> materials) - material_dirs = sorted([d for d in brand_dir.iterdir() if d.is_dir()]) - - for material_dir in material_dirs: - # Check if this is a material directory (has material.json) - if not (material_dir / 'material.json').exists(): - continue - - print(f" Material: {material_dir.name}") - - filament_dirs = sorted([d for d in material_dir.iterdir() if d.is_dir()]) - - for filament_dir in filament_dirs: - # Check if this is a filament directory (has filament.json) - if not (filament_dir / 'filament.json').exists(): - continue - - print(f" Filament: {filament_dir.name}") - - # Process variants FIRST (deepest level) - variant_dirs = sorted([d for d in filament_dir.iterdir() if d.is_dir()]) - for variant_dir in variant_dirs: - if (variant_dir / 'variant.json').exists(): - print(f" Variant: {variant_dir.name}") - changed, _ = migrate_variant( - variant_dir, dry_run, id_mapping, valid_store_ids - ) - if changed: - stats['variants'] += 1 - - # Then process filament - changed, _ = migrate_filament(filament_dir, dry_run) - if changed: - stats['filaments'] += 1 - - # Then process material - changed, _ = migrate_material(material_dir, dry_run, report) - if changed: - stats['materials'] += 1 - - # Finally, process the brand itself - changed, _ = migrate_brand(brand_dir, dry_run) - if changed: - stats['brands'] += 1 - - return stats - - -def process_stores_directory( - stores_dir: Path, dry_run: bool -) -> tuple[int, dict[str, str]]: - """Process all store files in the stores directory. - - Returns: - Tuple of (count of modified store files, old_id -> new_id mapping). - """ - count = 0 - id_mapping: dict[str, str] = {} - - # Collect directories first to avoid iteration issues - store_dirs = sorted([d for d in stores_dir.iterdir() if d.is_dir()]) - - for store_dir in store_dirs: - print(f"\nStore: {store_dir.name}") - changed, _ = migrate_store(store_dir, dry_run, id_mapping) - if changed: - count += 1 - - return count, id_mapping - - -def get_valid_store_ids(stores_dir: Path) -> set[str]: - """Scan stores directory and return all valid store IDs. - - Returns: - Set of valid store IDs (folder names that contain store.json). - """ - valid_ids = set() - if not stores_dir.exists(): - return valid_ids - - for store_dir in stores_dir.iterdir(): - if store_dir.is_dir() and (store_dir / 'store.json').exists(): - valid_ids.add(store_dir.name) - - return valid_ids - - -def find_matching_store_id( - old_store_id: str, - id_mapping: dict[str, str], - valid_store_ids: set[str] -) -> str | None: - """Find the correct store ID for an old store_id reference. - - Tries multiple strategies to find a match: - 1. Check explicit id_mapping from migration - 2. Check if it's already a valid store ID - 3. Check if generate_id(old_store_id) matches a valid store - 4. Check if any valid store ID contains the normalized old_store_id - - Args: - old_store_id: The store_id value found in data files. - id_mapping: Dictionary mapping old store IDs to new store IDs. - valid_store_ids: Set of valid store IDs from stores directory. - - Returns: - The matching store ID, or None if no match found. - """ - # 1. Check explicit mapping from migration - if old_store_id in id_mapping: - return id_mapping[old_store_id] - - # 2. Already a valid store ID - if old_store_id in valid_store_ids: - return old_store_id - - # 3. Check if normalized form is valid - normalized = generate_id(old_store_id) - if normalized in valid_store_ids: - return normalized - - # 4. Try fuzzy matching - check if removing common suffixes helps - # e.g., "3do.dk" -> check if "3do" exists - for suffix in ['.dk', '.com', '.de', '.eu', '_amazon', '_store']: - if old_store_id.lower().endswith(suffix): - base = old_store_id[:-len(suffix)] - base_normalized = generate_id(base) - if base_normalized in valid_store_ids: - return base_normalized - - # 5. Check case-insensitive match - old_lower = old_store_id.lower() - for valid_id in valid_store_ids: - if valid_id.lower() == old_lower: - return valid_id - - return None - - -def update_store_references( - data_dir: Path, - stores_dir: Path, - id_mapping: dict[str, str], - dry_run: bool, - report: MigrationReport -) -> int: - """Update store_id references and migrate spool_refill in all sizes.json files. - - Normalizes store_id values to match existing store IDs. - Uses multiple strategies to find the correct store ID: - 1. Explicit mappings discovered during store migration - 2. Direct match with existing store IDs - 3. Normalized form matching - 4. Fuzzy matching for common patterns - - Also migrates spool_refill from purchase_link level to size level. - - Args: - data_dir: Path to the data directory. - stores_dir: Path to the stores directory. - id_mapping: Dictionary mapping old store IDs to new store IDs - (built dynamically during store migration). - dry_run: If True, don't actually modify files. - report: MigrationReport to add issues to. - - Returns: - Count of sizes.json files updated. - """ - print(f"\nUpdating store_id references...") - - # Get all valid store IDs from the stores directory - valid_store_ids = get_valid_store_ids(stores_dir) - print(f" Found {len(valid_store_ids)} valid stores") - - if id_mapping: - print(f" Discovered mappings: {id_mapping}") - - count = 0 - unmatched_stores: set[str] = set() - - # Find all sizes.json files - for sizes_file in data_dir.rglob('sizes.json'): - data = load_json(sizes_file) - if data is None: - continue - - file_changed = False - - # sizes.json is a list of size objects - if isinstance(data, list): - # First, migrate spool_refill from purchase_link level to size level - data, spool_refill_changed = migrate_spool_refill(data) - if spool_refill_changed: - print(f" {sizes_file.relative_to(data_dir)}: migrated spool_refill") - file_changed = True - - # Then update store_id references - for size_obj in data: - if not isinstance(size_obj, dict): - continue - purchase_links = size_obj.get('purchase_links', []) - if not isinstance(purchase_links, list): - continue - - for link in purchase_links: - if not isinstance(link, dict): - continue - old_store_id = link.get('store_id') - if not old_store_id: - continue - - # Find the matching store ID - new_store_id = find_matching_store_id( - old_store_id, id_mapping, valid_store_ids - ) - - if new_store_id is None: - # No match found - track for reporting - unmatched_stores.add(old_store_id) - continue - - if old_store_id != new_store_id: - link['store_id'] = new_store_id - print(f" {sizes_file.relative_to(data_dir)}: " - f"{old_store_id} -> {new_store_id}") - file_changed = True - - if file_changed: - save_json(sizes_file, data, dry_run) - count += 1 - - # Report unmatched stores - for store_id in sorted(unmatched_stores): - report.add_issue( - "Unknown store reference", - "sizes.json files", - f"store_id '{store_id}' does not match any known store" - ) - - return count - - -def main(): - parser = argparse.ArgumentParser( - description='Migrate data files to new schema format' - ) - parser.add_argument( - '--dry-run', - action='store_true', - help='Preview changes without modifying files' - ) - parser.add_argument( - '--data-only', - action='store_true', - help='Only process data directory (skip stores)' - ) - parser.add_argument( - '--stores-only', - action='store_true', - help='Only process stores directory (skip data)' - ) - args = parser.parse_args() - - # Determine paths relative to script location - script_dir = Path(__file__).parent - repo_root = script_dir.parent - data_dir = repo_root / 'data' - stores_dir = repo_root / 'stores' - - if args.dry_run: - print("=== DRY RUN MODE - No files will be modified ===\n") - - total_changes = 0 - report = MigrationReport() - - # Process stores directory FIRST to build the ID mapping - # This mapping is needed to update store_id references in data files - store_id_mapping: dict[str, str] = {} - valid_store_ids: set[str] = set() - if not args.data_only: - if stores_dir.exists(): - print("Processing stores directory...") - store_count, store_id_mapping = process_stores_directory( - stores_dir, args.dry_run - ) - # Get valid store IDs after processing (folder names after migration) - valid_store_ids = get_valid_store_ids(stores_dir) - total_changes += store_count - print(f"\n--- Stores Summary ---") - print(f" Stores modified: {store_count}") - print(f" Valid store IDs: {len(valid_store_ids)}") - if store_id_mapping: - print(f" ID mappings discovered: {len(store_id_mapping)}") - else: - print(f"Stores directory not found: {stores_dir}") - else: - # Even if skipping stores, we need valid IDs to update references - valid_store_ids = get_valid_store_ids(stores_dir) - - # Process data directory (includes store_id reference updates during variant migration) - if not args.stores_only: - if data_dir.exists(): - print("\nProcessing data directory...") - stats = process_data_directory( - data_dir, args.dry_run, report, store_id_mapping, valid_store_ids - ) - total_changes += sum(stats.values()) - print(f"\n--- Data Summary ---") - print(f" Brands modified: {stats['brands']}") - print(f" Materials modified: {stats['materials']}") - print(f" Filaments modified: {stats['filaments']}") - print(f" Variants modified: {stats['variants']}") - else: - print(f"Data directory not found: {data_dir}") - - print(f"\n{'=' * 40}") - if args.dry_run: - print(f"DRY RUN COMPLETE: {total_changes} items would be modified") - else: - print(f"MIGRATION COMPLETE: {total_changes} items modified") - - # Print any issues encountered - report.print_summary() - - -if __name__ == '__main__': - main() From 0430df8475a14bbc49689c24f7dc92581bb0033e Mon Sep 17 00:00:00 2001 From: Frida Rosenaa Date: Sat, 10 Jan 2026 14:18:25 +0100 Subject: [PATCH 04/12] Allow functionality to sort data --- .github/workflows/sort_data.yaml | 61 ++++ scripts/sort_data.py | 503 +++++++++++++++++++++++++++++++ 2 files changed, 564 insertions(+) create mode 100644 .github/workflows/sort_data.yaml create mode 100644 scripts/sort_data.py diff --git a/.github/workflows/sort_data.yaml b/.github/workflows/sort_data.yaml new file mode 100644 index 0000000000..1cee14539c --- /dev/null +++ b/.github/workflows/sort_data.yaml @@ -0,0 +1,61 @@ +name: Sort Data + +on: + pull_request: + paths: + - 'data/**' + - 'stores/**' + - 'schemas/**' + - 'scripts/sort_data.py' + - 'data_validator.py' + - 'db_serializer.py' + - 'requirements.txt' + +jobs: + sort_data: + runs-on: ubuntu-24.04 + permissions: + contents: write + pull-requests: write + steps: + - name: Checkout PR branch + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup Git + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install dependencies + run: pip install -r requirements.txt + + - name: Run sort script + run: python scripts/sort_data.py + + - name: Check for changes + id: check_changes + run: | + if [[ -n $(git status --porcelain) ]]; then + echo "changes=true" >> $GITHUB_OUTPUT + else + echo "changes=false" >> $GITHUB_OUTPUT + fi + + - name: Commit and push changes + if: steps.check_changes.outputs.changes == 'true' + run: | + git add data/ stores/ + git commit -m "Auto-sort data files according to schema + + Automated sorting performed by sort_data.py script. + All changes have passed validation checks." + git push diff --git a/scripts/sort_data.py b/scripts/sort_data.py new file mode 100644 index 0000000000..487eed52b9 --- /dev/null +++ b/scripts/sort_data.py @@ -0,0 +1,503 @@ +#!/usr/bin/env python3 +""" +sort_data.py - Sort JSON file keys according to schema definitions + +This script recursively processes all JSON files in the data/ and stores/ +directories and reorders their keys to match the order defined in the +corresponding JSON schemas. This ensures consistent formatting across all +data files. + +The script: +1. Loads all schemas and extracts property key orderings +2. Processes each JSON file and sorts keys according to schema +3. Handles nested objects with their own key orderings +4. Warns about keys found in data but not in schema +5. Validates all files after sorting using data_validator.py + +Usage: + python scripts/sort_data.py --dry-run # Preview changes + python scripts/sort_data.py # Apply changes +""" + +import argparse +import json +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Optional, Set + +# Add parent directory to path to import data_validator +sys.path.insert(0, str(Path(__file__).parent.parent)) +from data_validator import ValidationOrchestrator + + +@dataclass +class SchemaInfo: + """Holds key ordering information for a schema.""" + keys: List[str] = field(default_factory=list) + nested: Dict[str, List[str]] = field(default_factory=dict) + + +@dataclass +class ProcessingStats: + """Statistics for file processing.""" + files_processed: int = 0 + files_modified: int = 0 + files_skipped: int = 0 + extra_keys_found: int = 0 + + +def load_json(path: Path) -> Optional[Dict[str, Any]]: + """Load JSON from file with error handling.""" + try: + with open(path, 'r', encoding='utf-8') as f: + return json.load(f) + except (json.JSONDecodeError, OSError) as e: + print(f"Error loading {path}: {e}") + return None + + +def save_json(path: Path, data: Any, dry_run: bool) -> None: + """Save JSON to file with consistent formatting.""" + if dry_run: + return + with open(path, 'w', encoding='utf-8') as f: + json.dump(data, f, indent=2, ensure_ascii=False) + f.write('\n') + + +def load_schemas(schemas_dir: Path) -> Dict[str, Dict[str, Any]]: + """Load all JSON schemas from the schemas directory.""" + schemas = {} + + if not schemas_dir.exists(): + print(f"Error: {schemas_dir} directory not found") + return schemas + + for schema_file in schemas_dir.glob("*.json"): + try: + with open(schema_file, 'r', encoding='utf-8') as f: + schemas[schema_file.stem] = json.load(f) + except json.JSONDecodeError as e: + print(f"Error parsing {schema_file.name}: {e}") + + return schemas + + +def get_property_order(schema: Dict[str, Any]) -> List[str]: + """Extract the order of properties from a JSON schema.""" + if "properties" in schema: + return list(schema["properties"].keys()) + return [] + + +def extract_nested_schemas(schema: Dict[str, Any]) -> Dict[str, List[str]]: + """Recursively extract nested object schemas and their key orderings. + + Returns: + Dictionary mapping nested property names to their key orderings. + For example: {'color_standards': ['ral', 'ncs', 'pantone', ...]} + """ + nested = {} + + # Check properties in the main schema + if "properties" in schema: + for prop_name, prop_schema in schema["properties"].items(): + if isinstance(prop_schema, dict): + # Handle object types + if prop_schema.get("type") == "object" and "properties" in prop_schema: + nested[prop_name] = get_property_order(prop_schema) + # Handle array types with object items + elif prop_schema.get("type") == "array" and "items" in prop_schema: + items_schema = prop_schema["items"] + if isinstance(items_schema, dict) and items_schema.get("type") == "object": + if "properties" in items_schema: + nested[prop_name] = get_property_order(items_schema) + + # Check definitions section for nested schemas + if "definitions" in schema: + for def_name, def_schema in schema["definitions"].items(): + if isinstance(def_schema, dict) and def_schema.get("type") == "object": + if "properties" in def_schema: + nested[def_name] = get_property_order(def_schema) + + return nested + + +def build_key_order_map(schemas_dir: Path) -> Dict[str, SchemaInfo]: + """Build a mapping of schema names to their key orderings. + + Returns: + Dictionary mapping schema names (without _schema suffix) to SchemaInfo. + """ + schemas = load_schemas(schemas_dir) + key_order_map = {} + + for schema_name, schema_content in schemas.items(): + # Remove _schema suffix if present + clean_name = schema_name.replace('_schema', '') + + # Check if this is an array schema (like sizes_schema.json) + if schema_content.get('type') == 'array' and 'items' in schema_content: + # For array schemas, use the items schema + items_schema = schema_content['items'] + keys = get_property_order(items_schema) + nested = extract_nested_schemas(items_schema) + else: + # Extract top-level key order + keys = get_property_order(schema_content) + # Extract nested object key orderings + nested = extract_nested_schemas(schema_content) + + key_order_map[clean_name] = SchemaInfo(keys=keys, nested=nested) + + return key_order_map + + +def sort_json_keys( + data: Any, + schema_info: SchemaInfo, + extra_keys: Set[str] +) -> Any: + """Recursively sort JSON keys according to schema ordering. + + Args: + data: The JSON data to sort (dict, list, or primitive) + schema_info: Schema information with key orderings + extra_keys: Set to collect keys not in schema + + Returns: + Sorted JSON data + """ + if isinstance(data, dict): + ordered = {} + remaining_keys = set(data.keys()) + + # Add keys in schema order first + for key in schema_info.keys: + if key in data: + value = data[key] + + # Check if this key has a nested schema + if key in schema_info.nested: + nested_info = SchemaInfo(keys=schema_info.nested[key], nested=schema_info.nested) + + # If value is an array, process each item with nested schema + if isinstance(value, list): + value = [sort_json_keys(item, nested_info, extra_keys) if isinstance(item, dict) + else item for item in value] + else: + value = sort_json_keys(value, nested_info, extra_keys) + elif isinstance(value, dict): + # No specific nested schema, just recurse with empty schema + value = sort_json_keys(value, SchemaInfo(), extra_keys) + elif isinstance(value, list): + # Process list items + value = [sort_json_keys(item, schema_info, extra_keys) if isinstance(item, dict) + else item for item in value] + + ordered[key] = value + remaining_keys.remove(key) + + # Add remaining keys alphabetically (these are not in schema) + if remaining_keys: + extra_keys.update(remaining_keys) + for key in sorted(remaining_keys): + value = data[key] + if isinstance(value, dict): + value = sort_json_keys(value, SchemaInfo(), extra_keys) + elif isinstance(value, list): + value = [sort_json_keys(item, SchemaInfo(), extra_keys) if isinstance(item, dict) + else item for item in value] + ordered[key] = value + + return ordered + + elif isinstance(data, list): + # For arrays, process each item + # For sizes.json, each item should follow the size item schema + result = [] + for item in data: + if isinstance(item, dict): + # For array items, use the same schema_info + result.append(sort_json_keys(item, schema_info, extra_keys)) + elif isinstance(item, list): + result.append(sort_json_keys(item, schema_info, extra_keys)) + else: + result.append(item) + return result + + else: + return data + + +def process_json_file( + file_path: Path, + schema_name: str, + key_order_map: Dict[str, SchemaInfo], + dry_run: bool, + stats: ProcessingStats +) -> bool: + """Process a single JSON file and sort its keys. + + Args: + file_path: Path to the JSON file + schema_name: Name of the schema to use + key_order_map: Mapping of schema names to key orderings + dry_run: If True, don't save changes + stats: Statistics tracker + + Returns: + True if file was modified, False otherwise + """ + # Load the file + data = load_json(file_path) + if data is None: + stats.files_skipped += 1 + return False + + # Get schema info + if schema_name not in key_order_map: + print(f" Warning: No schema found for {schema_name}") + stats.files_skipped += 1 + return False + + schema_info = key_order_map[schema_name] + + # Track extra keys found in this file + extra_keys: Set[str] = set() + + # Sort the keys + sorted_data = sort_json_keys(data, schema_info, extra_keys) + + # Warn about extra keys + if extra_keys: + print(f" Warning: Extra keys in {file_path.name}: {sorted(extra_keys)}") + stats.extra_keys_found += len(extra_keys) + + # Check if anything changed + original_json = json.dumps(data, ensure_ascii=False, sort_keys=False) + sorted_json = json.dumps(sorted_data, ensure_ascii=False, sort_keys=False) + + stats.files_processed += 1 + + if original_json != sorted_json: + if dry_run: + print(f" Would sort: {file_path.name}") + else: + print(f" Sorted: {file_path.name}") + save_json(file_path, sorted_data, dry_run) + stats.files_modified += 1 + return True + + return False + + +def process_data_directory( + data_dir: Path, + key_order_map: Dict[str, SchemaInfo], + dry_run: bool +) -> ProcessingStats: + """Process all JSON files in the data directory hierarchy. + + Args: + data_dir: Path to the data directory + key_order_map: Mapping of schema names to key orderings + dry_run: If True, don't save changes + + Returns: + Processing statistics + """ + stats = ProcessingStats() + + print("Processing data directory...") + + # Process each brand directory + for brand_dir in sorted(data_dir.iterdir()): + if not brand_dir.is_dir(): + continue + + print(f" Brand: {brand_dir.name}") + + # Process brand.json + brand_file = brand_dir / "brand.json" + if brand_file.exists(): + process_json_file(brand_file, "brand", key_order_map, dry_run, stats) + + # Process each material directory + for material_dir in sorted(brand_dir.iterdir()): + if not material_dir.is_dir(): + continue + + # Process material.json + material_file = material_dir / "material.json" + if material_file.exists(): + process_json_file(material_file, "material", key_order_map, dry_run, stats) + + # Process each filament directory + for filament_dir in sorted(material_dir.iterdir()): + if not filament_dir.is_dir(): + continue + + # Process filament.json + filament_file = filament_dir / "filament.json" + if filament_file.exists(): + process_json_file(filament_file, "filament", key_order_map, dry_run, stats) + + # Process each variant directory + for variant_dir in sorted(filament_dir.iterdir()): + if not variant_dir.is_dir(): + continue + + # Process variant.json + variant_file = variant_dir / "variant.json" + if variant_file.exists(): + process_json_file(variant_file, "variant", key_order_map, dry_run, stats) + + # Process sizes.json + sizes_file = variant_dir / "sizes.json" + if sizes_file.exists(): + process_json_file(sizes_file, "sizes", key_order_map, dry_run, stats) + + return stats + + +def process_stores_directory( + stores_dir: Path, + key_order_map: Dict[str, SchemaInfo], + dry_run: bool +) -> ProcessingStats: + """Process all JSON files in the stores directory. + + Args: + stores_dir: Path to the stores directory + key_order_map: Mapping of schema names to key orderings + dry_run: If True, don't save changes + + Returns: + Processing statistics + """ + stats = ProcessingStats() + + print("\nProcessing stores directory...") + + # Process each store directory + for store_dir in sorted(stores_dir.iterdir()): + if not store_dir.is_dir(): + continue + + print(f" Store: {store_dir.name}") + + # Process store.json + store_file = store_dir / "store.json" + if store_file.exists(): + process_json_file(store_file, "store", key_order_map, dry_run, stats) + + return stats + + +def merge_stats(stats1: ProcessingStats, stats2: ProcessingStats) -> ProcessingStats: + """Merge two ProcessingStats objects.""" + return ProcessingStats( + files_processed=stats1.files_processed + stats2.files_processed, + files_modified=stats1.files_modified + stats2.files_modified, + files_skipped=stats1.files_skipped + stats2.files_skipped, + extra_keys_found=stats1.extra_keys_found + stats2.extra_keys_found + ) + + +def main(): + parser = argparse.ArgumentParser( + description='Sort JSON file keys according to schema definitions' + ) + parser.add_argument( + '--dry-run', + action='store_true', + help='Preview changes without modifying files' + ) + args = parser.parse_args() + + # Determine paths relative to script location + script_dir = Path(__file__).parent + repo_root = script_dir.parent + data_dir = repo_root / 'data' + stores_dir = repo_root / 'stores' + schemas_dir = repo_root / 'schemas' + + if args.dry_run: + print("=== DRY RUN MODE - No files will be modified ===\n") + + # Build key order mapping from schemas + print("Loading schemas...") + key_order_map = build_key_order_map(schemas_dir) + print(f"Loaded {len(key_order_map)} schemas\n") + + # Process data directory + data_stats = ProcessingStats() + if data_dir.exists(): + data_stats = process_data_directory(data_dir, key_order_map, args.dry_run) + else: + print(f"Data directory not found: {data_dir}") + + # Process stores directory + stores_stats = ProcessingStats() + if stores_dir.exists(): + stores_stats = process_stores_directory(stores_dir, key_order_map, args.dry_run) + else: + print(f"Stores directory not found: {stores_dir}") + + # Merge statistics + total_stats = merge_stats(data_stats, stores_stats) + + # Print summary + print(f"\n{'=' * 60}") + if args.dry_run: + print("DRY RUN SUMMARY") + else: + print("SORTING SUMMARY") + print('=' * 60) + print(f"Files processed: {total_stats.files_processed}") + print(f"Files modified: {total_stats.files_modified}") + print(f"Files skipped: {total_stats.files_skipped}") + if total_stats.extra_keys_found > 0: + print(f"Extra keys found: {total_stats.extra_keys_found}") + + # Run validation if not in dry-run mode + if not args.dry_run and total_stats.files_modified > 0: + print(f"\n{'=' * 60}") + print("VALIDATING SORTED FILES") + print('=' * 60) + + orchestrator = ValidationOrchestrator(data_dir, stores_dir) + result = orchestrator.validate_all() + + if result.errors: + print("\nValidation errors found:") + + # Group errors by category + errors_by_category: Dict[str, List] = {} + for error in result.errors: + if error.category not in errors_by_category: + errors_by_category[error.category] = [] + errors_by_category[error.category].append(error) + + # Print errors grouped by category + for category, errors in sorted(errors_by_category.items()): + print(f"\n{category} ({len(errors)}):") + print("-" * 80) + for error in errors[:10]: # Limit to first 10 per category + print(f" {error}") + if len(errors) > 10: + print(f" ... and {len(errors) - 10} more") + + print(f"\nValidation failed: {result.error_count} errors, {result.warning_count} warnings") + sys.exit(1) + else: + print("\nAll validations passed!") + + print("\nDone!") + sys.exit(0) + + +if __name__ == '__main__': + main() From a4eccd8ad32c47dc290e32622f00a3983be9dce2 Mon Sep 17 00:00:00 2001 From: Frida Rosenaa Date: Sat, 10 Jan 2026 14:25:13 +0100 Subject: [PATCH 05/12] update to allow running on remote repos, so for anyone except for maintainers... --- .github/workflows/sort_data.yaml | 33 +++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/.github/workflows/sort_data.yaml b/.github/workflows/sort_data.yaml index 1cee14539c..51cb9bdd4a 100644 --- a/.github/workflows/sort_data.yaml +++ b/.github/workflows/sort_data.yaml @@ -1,15 +1,12 @@ name: Sort Data on: - pull_request: + pull_request_target: + types: [opened, synchronize, reopened] paths: - 'data/**' - 'stores/**' - 'schemas/**' - - 'scripts/sort_data.py' - - 'data_validator.py' - - 'db_serializer.py' - - 'requirements.txt' jobs: sort_data: @@ -18,12 +15,21 @@ jobs: contents: write pull-requests: write steps: + - name: Generate token for pushing to forks + id: generate_token + uses: tibdex/github-app-token@v2 + with: + app_id: ${{ secrets.SORT_BOT_APP_ID }} + private_key: ${{ secrets.SORT_BOT_PRIVATE_KEY }} + continue-on-error: true + - name: Checkout PR branch uses: actions/checkout@v4 with: - ref: ${{ github.event.pull_request.head.ref }} repository: ${{ github.event.pull_request.head.repo.full_name }} - token: ${{ secrets.GITHUB_TOKEN }} + ref: ${{ github.event.pull_request.head.ref }} + token: ${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }} + persist-credentials: true - name: Setup Git run: | @@ -59,3 +65,16 @@ jobs: Automated sorting performed by sort_data.py script. All changes have passed validation checks." git push + continue-on-error: true + + - name: Comment on PR if push failed + if: steps.check_changes.outputs.changes == 'true' && failure() + uses: actions/github-script@v7 + with: + script: | + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: '⚠️ **Data sorting required**\n\nThis PR contains unsorted data files. Please run the following command locally and push the changes:\n\n```bash\npython scripts/sort_data.py\n```\n\nThis will sort all JSON files according to the schema definitions and validate the results.' + }) From bbe98fc0b7e45363c21213f7ad2e5877fc16ef70 Mon Sep 17 00:00:00 2001 From: Frida Rosenaa Date: Sat, 10 Jan 2026 14:26:43 +0100 Subject: [PATCH 06/12] Remove workflow, we'll use the webui to enforce usage --- .github/workflows/sort_data.yaml | 80 -------------------------------- 1 file changed, 80 deletions(-) delete mode 100644 .github/workflows/sort_data.yaml diff --git a/.github/workflows/sort_data.yaml b/.github/workflows/sort_data.yaml deleted file mode 100644 index 51cb9bdd4a..0000000000 --- a/.github/workflows/sort_data.yaml +++ /dev/null @@ -1,80 +0,0 @@ -name: Sort Data - -on: - pull_request_target: - types: [opened, synchronize, reopened] - paths: - - 'data/**' - - 'stores/**' - - 'schemas/**' - -jobs: - sort_data: - runs-on: ubuntu-24.04 - permissions: - contents: write - pull-requests: write - steps: - - name: Generate token for pushing to forks - id: generate_token - uses: tibdex/github-app-token@v2 - with: - app_id: ${{ secrets.SORT_BOT_APP_ID }} - private_key: ${{ secrets.SORT_BOT_PRIVATE_KEY }} - continue-on-error: true - - - name: Checkout PR branch - uses: actions/checkout@v4 - with: - repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.ref }} - token: ${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }} - persist-credentials: true - - - name: Setup Git - run: | - git config --global user.name "github-actions[bot]" - git config --global user.email "github-actions[bot]@users.noreply.github.com" - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.11' - - - name: Install dependencies - run: pip install -r requirements.txt - - - name: Run sort script - run: python scripts/sort_data.py - - - name: Check for changes - id: check_changes - run: | - if [[ -n $(git status --porcelain) ]]; then - echo "changes=true" >> $GITHUB_OUTPUT - else - echo "changes=false" >> $GITHUB_OUTPUT - fi - - - name: Commit and push changes - if: steps.check_changes.outputs.changes == 'true' - run: | - git add data/ stores/ - git commit -m "Auto-sort data files according to schema - - Automated sorting performed by sort_data.py script. - All changes have passed validation checks." - git push - continue-on-error: true - - - name: Comment on PR if push failed - if: steps.check_changes.outputs.changes == 'true' && failure() - uses: actions/github-script@v7 - with: - script: | - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: '⚠️ **Data sorting required**\n\nThis PR contains unsorted data files. Please run the following command locally and push the changes:\n\n```bash\npython scripts/sort_data.py\n```\n\nThis will sort all JSON files according to the schema definitions and validate the results.' - }) From 2ec872e183d94c5e9afbcdfaf3c34bb87b04773c Mon Sep 17 00:00:00 2001 From: Frida Rosenaa Date: Sat, 10 Jan 2026 14:57:54 +0100 Subject: [PATCH 07/12] Allow systemic outputs of python scripts, for webui --- data_validator.py | 108 ++++++++++++++++++++------ scripts/sort_data.py | 176 +++++++++++++++++++++++++++++++------------ 2 files changed, 210 insertions(+), 74 deletions(-) diff --git a/data_validator.py b/data_validator.py index 8a76ae611e..325f57bed1 100644 --- a/data_validator.py +++ b/data_validator.py @@ -1,6 +1,7 @@ import json import re import os +import sys from concurrent.futures import ProcessPoolExecutor, as_completed from dataclasses import dataclass, field from enum import Enum @@ -49,6 +50,15 @@ def __str__(self) -> str: path_str = f" [{self.path}]" if self.path else "" return f"{self.level.value} - {self.category}: {self.message}{path_str}" + def to_dict(self) -> Dict[str, Any]: + """Convert to dictionary for JSON serialization.""" + return { + 'level': self.level.value, + 'category': self.category, + 'message': self.message, + 'path': str(self.path) if self.path else None + } + @dataclass class ValidationResult: @@ -73,6 +83,15 @@ def error_count(self) -> int: def warning_count(self) -> int: return len([e for e in self.errors if e.level == ValidationLevel.WARNING]) + def to_dict(self) -> Dict[str, Any]: + """Convert to dictionary for JSON serialization.""" + return { + 'errors': [e.to_dict() for e in self.errors], + 'error_count': self.error_count, + 'warning_count': self.warning_count, + 'is_valid': self.is_valid + } + # ------------------------- # Utility Functions @@ -781,11 +800,24 @@ class ValidationOrchestrator: def __init__(self, data_dir: Path = Path("./data"), stores_dir: Path = Path("./stores"), - max_workers: Optional[int] = None): + max_workers: Optional[int] = None, + progress_mode: bool = False): self.data_dir = data_dir self.stores_dir = stores_dir self.max_workers = max_workers self.schema_cache = SchemaCache() + self.progress_mode = progress_mode + + def emit_progress(self, stage: str, percent: int, message: str = '') -> None: + """Emit progress event as JSON to stdout for SSE streaming.""" + if self.progress_mode and hasattr(sys.stdout, 'isatty') and not sys.stdout.isatty(): + # Only emit when stdout is piped (not terminal) + print(json.dumps({ + 'type': 'progress', + 'stage': stage, + 'percent': percent, + 'message': message + }), flush=True) def run_tasks_parallel(self, tasks: List[ValidationTask]) -> ValidationResult: """Run validation tasks in parallel using process pool.""" @@ -850,15 +882,31 @@ def validate_all(self) -> ValidationResult: result = ValidationResult() # Check for missing files first + self.emit_progress('missing_files', 0, 'Checking for missing required files...') print("Checking for missing required files...") validator = MissingFileValidator(self.schema_cache) result.merge(validator.validate_required_files(self.data_dir, self.stores_dir)) + self.emit_progress('missing_files', 100, 'Missing files check complete') + self.emit_progress('json_files', 0, 'Validating JSON files...') result.merge(self.validate_json_files()) + self.emit_progress('json_files', 100, 'JSON validation complete') + + self.emit_progress('logo_files', 0, 'Validating logo files...') result.merge(self.validate_logo_files()) + self.emit_progress('logo_files', 100, 'Logo validation complete') + + self.emit_progress('folder_names', 0, 'Validating folder names...') result.merge(self.validate_folder_names()) + self.emit_progress('folder_names', 100, 'Folder name validation complete') + + self.emit_progress('store_ids', 0, 'Validating store IDs...') result.merge(self.validate_store_ids()) + self.emit_progress('store_ids', 100, 'Store ID validation complete') + + self.emit_progress('gtin', 0, 'Validating GTIN/EAN...') result.merge(self.validate_gtin()) + self.emit_progress('gtin', 100, 'GTIN/EAN validation complete') return result @@ -876,15 +924,18 @@ def main(): parser.add_argument("--folder-names", action="store_true", help="Validate folder names") parser.add_argument("--store-ids", action="store_true", help="Validate store IDs") + parser.add_argument("--json", action="store_true", help="Output results as JSON") + parser.add_argument("--progress", action="store_true", help="Emit progress events (for SSE streaming)") args = parser.parse_args() - orchestrator = ValidationOrchestrator(max_workers=os.cpu_count()) + orchestrator = ValidationOrchestrator(max_workers=os.cpu_count(), progress_mode=args.progress) result = ValidationResult() # Run requested validations - if not any(vars(args).values()): - print("No args passed, validating all") + if not any([args.json_files, args.logo_files, args.folder_names, args.store_ids]): + if not args.json and not args.progress: + print("No args passed, validating all") result = orchestrator.validate_all() else: if args.json_files: @@ -896,28 +947,35 @@ def main(): if args.store_ids: result.merge(orchestrator.validate_store_ids()) - # Print results - if result.errors: - # Group errors by category - errors_by_category: Dict[str, List[ValidationError]] = {} - for error in result.errors: - if error.category not in errors_by_category: - errors_by_category[error.category] = [] - errors_by_category[error.category].append(error) - - # Print errors grouped by category - for category, errors in sorted(errors_by_category.items()): - print(f"\n{category} ({len(errors)}):") - print("-" * 80) - for error in errors: - print(f" {error}") - - print( - f"\nValidation failed: {result.error_count} errors, {result.warning_count} warnings") - exit(1) + # Output results + if args.json: + # JSON output mode + output = result.to_dict() + print(json.dumps(output, indent=2)) + sys.exit(0 if result.is_valid else 1) else: - print("All validations passed!") - exit(0) + # Text output mode + if result.errors: + # Group errors by category + errors_by_category: Dict[str, List[ValidationError]] = {} + for error in result.errors: + if error.category not in errors_by_category: + errors_by_category[error.category] = [] + errors_by_category[error.category].append(error) + + # Print errors grouped by category + for category, errors in sorted(errors_by_category.items()): + print(f"\n{category} ({len(errors)}):") + print("-" * 80) + for error in errors: + print(f" {error}") + + print( + f"\nValidation failed: {result.error_count} errors, {result.warning_count} warnings") + exit(1) + else: + print("All validations passed!") + exit(0) if __name__ == '__main__': diff --git a/scripts/sort_data.py b/scripts/sort_data.py index 487eed52b9..6635a50880 100644 --- a/scripts/sort_data.py +++ b/scripts/sort_data.py @@ -30,6 +30,22 @@ sys.path.insert(0, str(Path(__file__).parent.parent)) from data_validator import ValidationOrchestrator +# Global flags for output mode +PROGRESS_MODE = False +JSON_MODE = False + + +def emit_progress(stage: str, percent: int, message: str = '') -> None: + """Emit progress event as JSON to stdout for SSE streaming.""" + if PROGRESS_MODE and hasattr(sys.stdout, 'isatty') and not sys.stdout.isatty(): + # Only emit when stdout is piped (not terminal) + print(json.dumps({ + 'type': 'progress', + 'stage': stage, + 'percent': percent, + 'message': message + }), flush=True) + @dataclass class SchemaInfo: @@ -46,6 +62,15 @@ class ProcessingStats: files_skipped: int = 0 extra_keys_found: int = 0 + def to_dict(self) -> Dict[str, int]: + """Convert to dictionary for JSON serialization.""" + return { + 'files_processed': self.files_processed, + 'files_modified': self.files_modified, + 'files_skipped': self.files_skipped, + 'extra_keys_found': self.extra_keys_found + } + def load_json(path: Path) -> Optional[Dict[str, Any]]: """Load JSON from file with error handling.""" @@ -407,6 +432,8 @@ def merge_stats(stats1: ProcessingStats, stats2: ProcessingStats) -> ProcessingS def main(): + global PROGRESS_MODE, JSON_MODE + parser = argparse.ArgumentParser( description='Sort JSON file keys according to schema definitions' ) @@ -415,8 +442,26 @@ def main(): action='store_true', help='Preview changes without modifying files' ) + parser.add_argument( + '--json', + action='store_true', + help='Output results as JSON' + ) + parser.add_argument( + '--progress', + action='store_true', + help='Emit progress events (for SSE streaming)' + ) + parser.add_argument( + '--validate', + action='store_true', + help='Run validation after sorting' + ) args = parser.parse_args() + PROGRESS_MODE = args.progress + JSON_MODE = args.json + # Determine paths relative to script location script_dir = Path(__file__).parent repo_root = script_dir.parent @@ -425,78 +470,111 @@ def main(): schemas_dir = repo_root / 'schemas' if args.dry_run: - print("=== DRY RUN MODE - No files will be modified ===\n") + if not JSON_MODE: + print("=== DRY RUN MODE - No files will be modified ===\n") # Build key order mapping from schemas - print("Loading schemas...") + emit_progress('loading_schemas', 0, 'Loading schemas...') + if not JSON_MODE: + print("Loading schemas...") key_order_map = build_key_order_map(schemas_dir) - print(f"Loaded {len(key_order_map)} schemas\n") + emit_progress('loading_schemas', 100, f'Loaded {len(key_order_map)} schemas') + if not JSON_MODE: + print(f"Loaded {len(key_order_map)} schemas\n") # Process data directory data_stats = ProcessingStats() if data_dir.exists(): + emit_progress('sorting_data', 0, 'Processing data directory...') data_stats = process_data_directory(data_dir, key_order_map, args.dry_run) + emit_progress('sorting_data', 100, 'Data directory processing complete') else: - print(f"Data directory not found: {data_dir}") + if not JSON_MODE: + print(f"Data directory not found: {data_dir}") # Process stores directory stores_stats = ProcessingStats() if stores_dir.exists(): + emit_progress('sorting_stores', 0, 'Processing stores directory...') stores_stats = process_stores_directory(stores_dir, key_order_map, args.dry_run) + emit_progress('sorting_stores', 100, 'Stores directory processing complete') else: - print(f"Stores directory not found: {stores_dir}") + if not JSON_MODE: + print(f"Stores directory not found: {stores_dir}") # Merge statistics total_stats = merge_stats(data_stats, stores_stats) - # Print summary - print(f"\n{'=' * 60}") - if args.dry_run: - print("DRY RUN SUMMARY") + # Run validation if requested + validation_result = None + if args.validate and not args.dry_run and total_stats.files_modified > 0: + emit_progress('validation', 0, 'Running validation...') + if not JSON_MODE: + print(f"\n{'=' * 60}") + print("VALIDATING SORTED FILES") + print('=' * 60) + + orchestrator = ValidationOrchestrator(data_dir, stores_dir, progress_mode=PROGRESS_MODE) + validation_result = orchestrator.validate_all() + emit_progress('validation', 100, 'Validation complete') + + # Output results + if JSON_MODE: + # JSON output mode + output = { + 'dry_run': args.dry_run, + 'stats': total_stats.to_dict() + } + if validation_result: + output['validation'] = validation_result.to_dict() + print(json.dumps(output, indent=2)) + + # Exit with error code if validation failed + if validation_result and not validation_result.is_valid: + sys.exit(1) + sys.exit(0) else: - print("SORTING SUMMARY") - print('=' * 60) - print(f"Files processed: {total_stats.files_processed}") - print(f"Files modified: {total_stats.files_modified}") - print(f"Files skipped: {total_stats.files_skipped}") - if total_stats.extra_keys_found > 0: - print(f"Extra keys found: {total_stats.extra_keys_found}") - - # Run validation if not in dry-run mode - if not args.dry_run and total_stats.files_modified > 0: + # Text output mode print(f"\n{'=' * 60}") - print("VALIDATING SORTED FILES") - print('=' * 60) - - orchestrator = ValidationOrchestrator(data_dir, stores_dir) - result = orchestrator.validate_all() - - if result.errors: - print("\nValidation errors found:") - - # Group errors by category - errors_by_category: Dict[str, List] = {} - for error in result.errors: - if error.category not in errors_by_category: - errors_by_category[error.category] = [] - errors_by_category[error.category].append(error) - - # Print errors grouped by category - for category, errors in sorted(errors_by_category.items()): - print(f"\n{category} ({len(errors)}):") - print("-" * 80) - for error in errors[:10]: # Limit to first 10 per category - print(f" {error}") - if len(errors) > 10: - print(f" ... and {len(errors) - 10} more") - - print(f"\nValidation failed: {result.error_count} errors, {result.warning_count} warnings") - sys.exit(1) + if args.dry_run: + print("DRY RUN SUMMARY") else: - print("\nAll validations passed!") + print("SORTING SUMMARY") + print('=' * 60) + print(f"Files processed: {total_stats.files_processed}") + print(f"Files modified: {total_stats.files_modified}") + print(f"Files skipped: {total_stats.files_skipped}") + if total_stats.extra_keys_found > 0: + print(f"Extra keys found: {total_stats.extra_keys_found}") + + if validation_result: + if validation_result.errors: + print("\nValidation errors found:") + + # Group errors by category + errors_by_category: Dict[str, List] = {} + for error in validation_result.errors: + if error.category not in errors_by_category: + errors_by_category[error.category] = [] + errors_by_category[error.category].append(error) + + # Print errors grouped by category + for category, errors in sorted(errors_by_category.items()): + print(f"\n{category} ({len(errors)}):") + print("-" * 80) + for error in errors[:10]: # Limit to first 10 per category + print(f" {error}") + if len(errors) > 10: + print(f" ... and {len(errors) - 10} more") + + print(f"\nValidation failed: {validation_result.error_count} errors, {validation_result.warning_count} warnings") + print("\nDone!") + sys.exit(1) + else: + print("\nAll validations passed!") - print("\nDone!") - sys.exit(0) + print("\nDone!") + sys.exit(0) if __name__ == '__main__': From 32dddd7aea30cdd54d6629263b35c430f6748fb4 Mon Sep 17 00:00:00 2001 From: Frida Rosenaa Date: Sat, 10 Jan 2026 15:33:24 +0100 Subject: [PATCH 08/12] Add validation and sorting to webui --- data_validator.py | 33 +++-- scripts/sort_data.py | 34 +++-- .../src/lib/components/SortDataButton.svelte | 72 +++++++++++ .../lib/components/ValidationDropdown.svelte | 113 +++++++++++++++++ .../components/ValidationProgressModal.svelte | 106 ++++++++++++++++ .../components/ValidationTriggerButton.svelte | 72 +++++++++++ webui/src/lib/hooks/useSSE.ts | 49 ++++++++ webui/src/lib/server/jobManager.ts | 25 ++++ webui/src/lib/stores/validationStore.ts | 74 +++++++++++ webui/src/lib/utils/pathToRoute.ts | 48 +++++++ webui/src/routes/+layout.svelte | 17 ++- webui/src/routes/api/sort/+server.ts | 116 +++++++++++++++++ .../routes/api/sort/stream/[jobId]/+server.ts | 69 ++++++++++ webui/src/routes/api/validate/+server.ts | 118 ++++++++++++++++++ .../api/validate/stream/[jobId]/+server.ts | 69 ++++++++++ 15 files changed, 991 insertions(+), 24 deletions(-) create mode 100644 webui/src/lib/components/SortDataButton.svelte create mode 100644 webui/src/lib/components/ValidationDropdown.svelte create mode 100644 webui/src/lib/components/ValidationProgressModal.svelte create mode 100644 webui/src/lib/components/ValidationTriggerButton.svelte create mode 100644 webui/src/lib/hooks/useSSE.ts create mode 100644 webui/src/lib/server/jobManager.ts create mode 100644 webui/src/lib/stores/validationStore.ts create mode 100644 webui/src/lib/utils/pathToRoute.ts create mode 100644 webui/src/routes/api/sort/+server.ts create mode 100644 webui/src/routes/api/sort/stream/[jobId]/+server.ts create mode 100644 webui/src/routes/api/validate/+server.ts create mode 100644 webui/src/routes/api/validate/stream/[jobId]/+server.ts diff --git a/data_validator.py b/data_validator.py index 325f57bed1..259863976d 100644 --- a/data_validator.py +++ b/data_validator.py @@ -846,34 +846,42 @@ def run_tasks_parallel(self, tasks: List[ValidationTask]) -> ValidationResult: def validate_json_files(self) -> ValidationResult: """Validate all JSON files against schemas.""" - print("Collecting JSON validation tasks...") + if not self.progress_mode: + print("Collecting JSON validation tasks...") tasks = collect_json_validation_tasks(self.data_dir, self.stores_dir) - print(f"Running {len(tasks)} JSON validation tasks...") + if not self.progress_mode: + print(f"Running {len(tasks)} JSON validation tasks...") return self.run_tasks_parallel(tasks) def validate_logo_files(self) -> ValidationResult: """Validate all logo files.""" - print("Collecting logo validation tasks...") + if not self.progress_mode: + print("Collecting logo validation tasks...") tasks = collect_logo_validation_tasks(self.data_dir, self.stores_dir) - print(f"Running {len(tasks)} logo validation tasks...") + if not self.progress_mode: + print(f"Running {len(tasks)} logo validation tasks...") return self.run_tasks_parallel(tasks) def validate_folder_names(self) -> ValidationResult: """Validate all folder names.""" - print("Collecting folder name validation tasks...") + if not self.progress_mode: + print("Collecting folder name validation tasks...") tasks = collect_folder_validation_tasks(self.data_dir, self.stores_dir) - print(f"Running {len(tasks)} folder name validation tasks...") + if not self.progress_mode: + print(f"Running {len(tasks)} folder name validation tasks...") return self.run_tasks_parallel(tasks) def validate_store_ids(self) -> ValidationResult: """Validate store IDs.""" - print("Validating store IDs...") + if not self.progress_mode: + print("Validating store IDs...") validator = StoreIdValidator(self.schema_cache) return validator.validate_store_ids(self.data_dir, self.stores_dir) def validate_gtin(self) -> ValidationResult: """Validate GTIN/EAN rules.""" - print("Validating GTIN/EAN...") + if not self.progress_mode: + print("Validating GTIN/EAN...") validator = GTINValidator(self.schema_cache) return validator.validate_gtin_ean(self.data_dir) @@ -883,7 +891,8 @@ def validate_all(self) -> ValidationResult: # Check for missing files first self.emit_progress('missing_files', 0, 'Checking for missing required files...') - print("Checking for missing required files...") + if not self.progress_mode: + print("Checking for missing required files...") validator = MissingFileValidator(self.schema_cache) result.merge(validator.validate_required_files(self.data_dir, self.stores_dir)) self.emit_progress('missing_files', 100, 'Missing files check complete') @@ -951,7 +960,11 @@ def main(): if args.json: # JSON output mode output = result.to_dict() - print(json.dumps(output, indent=2)) + # Use compact output in progress mode for SSE compatibility, pretty output otherwise + if args.progress: + print(json.dumps(output)) + else: + print(json.dumps(output, indent=2)) sys.exit(0 if result.is_valid else 1) else: # Text output mode diff --git a/scripts/sort_data.py b/scripts/sort_data.py index 6635a50880..fcce8b7024 100644 --- a/scripts/sort_data.py +++ b/scripts/sort_data.py @@ -283,7 +283,8 @@ def process_json_file( # Get schema info if schema_name not in key_order_map: - print(f" Warning: No schema found for {schema_name}") + if not JSON_MODE: + print(f" Warning: No schema found for {schema_name}") stats.files_skipped += 1 return False @@ -297,7 +298,8 @@ def process_json_file( # Warn about extra keys if extra_keys: - print(f" Warning: Extra keys in {file_path.name}: {sorted(extra_keys)}") + if not JSON_MODE: + print(f" Warning: Extra keys in {file_path.name}: {sorted(extra_keys)}") stats.extra_keys_found += len(extra_keys) # Check if anything changed @@ -307,10 +309,11 @@ def process_json_file( stats.files_processed += 1 if original_json != sorted_json: - if dry_run: - print(f" Would sort: {file_path.name}") - else: - print(f" Sorted: {file_path.name}") + if not JSON_MODE: + if dry_run: + print(f" Would sort: {file_path.name}") + else: + print(f" Sorted: {file_path.name}") save_json(file_path, sorted_data, dry_run) stats.files_modified += 1 return True @@ -335,14 +338,16 @@ def process_data_directory( """ stats = ProcessingStats() - print("Processing data directory...") + if not JSON_MODE: + print("Processing data directory...") # Process each brand directory for brand_dir in sorted(data_dir.iterdir()): if not brand_dir.is_dir(): continue - print(f" Brand: {brand_dir.name}") + if not JSON_MODE: + print(f" Brand: {brand_dir.name}") # Process brand.json brand_file = brand_dir / "brand.json" @@ -404,14 +409,16 @@ def process_stores_directory( """ stats = ProcessingStats() - print("\nProcessing stores directory...") + if not JSON_MODE: + print("\nProcessing stores directory...") # Process each store directory for store_dir in sorted(stores_dir.iterdir()): if not store_dir.is_dir(): continue - print(f" Store: {store_dir.name}") + if not JSON_MODE: + print(f" Store: {store_dir.name}") # Process store.json store_file = store_dir / "store.json" @@ -527,7 +534,12 @@ def main(): } if validation_result: output['validation'] = validation_result.to_dict() - print(json.dumps(output, indent=2)) + + # Use compact output in progress mode for SSE compatibility, pretty output otherwise + if PROGRESS_MODE: + print(json.dumps(output)) + else: + print(json.dumps(output, indent=2)) # Exit with error code if validation failed if validation_result and not validation_result.is_valid: diff --git a/webui/src/lib/components/SortDataButton.svelte b/webui/src/lib/components/SortDataButton.svelte new file mode 100644 index 0000000000..63bb5ff077 --- /dev/null +++ b/webui/src/lib/components/SortDataButton.svelte @@ -0,0 +1,72 @@ + + + + + diff --git a/webui/src/lib/components/ValidationDropdown.svelte b/webui/src/lib/components/ValidationDropdown.svelte new file mode 100644 index 0000000000..953fc7c91d --- /dev/null +++ b/webui/src/lib/components/ValidationDropdown.svelte @@ -0,0 +1,113 @@ + + + + +
+ + + {#if isOpen} +
+ {#if $errorCount + $warningCount === 0} +
+ + + +

No validation issues

+
+ {:else} + {#each [...$errorsByCategory.entries()] as [category, errors]} +
+
+

+ {category} + ({errors.length}) +

+
+
+ {#each errors.slice(0, 5) as error} + + {/each} + {#if errors.length > 5} +
+ ... and {errors.length - 5} more +
+ {/if} +
+
+ {/each} + {/if} +
+ {/if} +
diff --git a/webui/src/lib/components/ValidationProgressModal.svelte b/webui/src/lib/components/ValidationProgressModal.svelte new file mode 100644 index 0000000000..71a7cf61dd --- /dev/null +++ b/webui/src/lib/components/ValidationProgressModal.svelte @@ -0,0 +1,106 @@ + + +{#if isOpen} +
+
e.stopPropagation()} + role="dialog" + tabindex="-1" + > +

+ {jobType === 'validation' ? 'Running Validation...' : 'Sorting Data...'} +

+ + {#if error} +
+ Error: + {error} +
+ {:else} +
+
+
+
+

+ {progress.stage || 'Initializing...'} +

+ {#if progress.message} +

{progress.message}

+ {/if} +
+ +
+
+
+
+ {/if} + + +
+
+{/if} diff --git a/webui/src/lib/components/ValidationTriggerButton.svelte b/webui/src/lib/components/ValidationTriggerButton.svelte new file mode 100644 index 0000000000..c1bb2c5da1 --- /dev/null +++ b/webui/src/lib/components/ValidationTriggerButton.svelte @@ -0,0 +1,72 @@ + + + + + diff --git a/webui/src/lib/hooks/useSSE.ts b/webui/src/lib/hooks/useSSE.ts new file mode 100644 index 0000000000..e213ae58d7 --- /dev/null +++ b/webui/src/lib/hooks/useSSE.ts @@ -0,0 +1,49 @@ +export interface SSEHandlers { + onProgress?: (data: any) => void; + onComplete?: (data: any) => void; + onError?: (error: any) => void; +} + +export function useSSE() { + let eventSource: EventSource | null = null; + + function connect(url: string, handlers: SSEHandlers) { + // Close any existing connection + disconnect(); + + eventSource = new EventSource(url); + + eventSource.onmessage = (event) => { + try { + const data = JSON.parse(event.data); + + if (data.type === 'progress' && handlers.onProgress) { + handlers.onProgress(data); + } else if (data.type === 'complete' && handlers.onComplete) { + handlers.onComplete(data.result); + disconnect(); + } else if (data.type === 'error' && handlers.onError) { + handlers.onError(data); + disconnect(); + } + } catch (e) { + console.error('SSE parse error:', e); + } + }; + + eventSource.onerror = (error) => { + console.error('SSE connection error:', error); + handlers.onError?.(error); + disconnect(); + }; + } + + function disconnect() { + if (eventSource) { + eventSource.close(); + eventSource = null; + } + } + + return { connect, disconnect }; +} diff --git a/webui/src/lib/server/jobManager.ts b/webui/src/lib/server/jobManager.ts new file mode 100644 index 0000000000..b83fa26758 --- /dev/null +++ b/webui/src/lib/server/jobManager.ts @@ -0,0 +1,25 @@ +// Shared job storage for validation and sort operations + +export interface Job { + id: string; + type: 'validation' | 'sort'; + startTime: number; + status: 'running' | 'complete' | 'error'; + events: any[]; + process?: any; + result?: any; + endTime?: number; +} + +// In-memory job storage +export const activeJobs = new Map(); + +// Cleanup old jobs (older than 5 minutes) +setInterval(() => { + const now = Date.now(); + for (const [jobId, job] of activeJobs.entries()) { + if (job.endTime && now - job.endTime > 5 * 60 * 1000) { + activeJobs.delete(jobId); + } + } +}, 60 * 1000); // Run every minute diff --git a/webui/src/lib/stores/validationStore.ts b/webui/src/lib/stores/validationStore.ts new file mode 100644 index 0000000000..8d7ecd8a91 --- /dev/null +++ b/webui/src/lib/stores/validationStore.ts @@ -0,0 +1,74 @@ +import { writable, derived } from 'svelte/store'; + +export interface ValidationError { + level: 'ERROR' | 'WARNING'; + category: string; + message: string; + path: string | null; +} + +export interface ValidationResult { + errors: ValidationError[]; + error_count: number; + warning_count: number; + is_valid: boolean; +} + +interface ValidationState { + isValidating: boolean; + lastValidation: Date | null; + results: ValidationResult | null; +} + +function createValidationStore() { + const { subscribe, set, update } = writable({ + isValidating: false, + lastValidation: null, + results: null + }); + + return { + subscribe, + setValidating: (isValidating: boolean) => + update((state) => ({ + ...state, + isValidating + })), + setResults: (results: ValidationResult) => + update((state) => ({ + ...state, + isValidating: false, + lastValidation: new Date(), + results + })), + clear: () => + set({ + isValidating: false, + lastValidation: null, + results: null + }) + }; +} + +export const validationStore = createValidationStore(); + +// Derived stores for convenience +export const errorCount = derived(validationStore, ($store) => $store.results?.error_count ?? 0); + +export const warningCount = derived( + validationStore, + ($store) => $store.results?.warning_count ?? 0 +); + +export const errorsByCategory = derived(validationStore, ($store) => { + if (!$store.results) return new Map(); + + const grouped = new Map(); + for (const error of $store.results.errors) { + if (!grouped.has(error.category)) { + grouped.set(error.category, []); + } + grouped.get(error.category)!.push(error); + } + return grouped; +}); diff --git a/webui/src/lib/utils/pathToRoute.ts b/webui/src/lib/utils/pathToRoute.ts new file mode 100644 index 0000000000..2afb443acf --- /dev/null +++ b/webui/src/lib/utils/pathToRoute.ts @@ -0,0 +1,48 @@ +/** + * Convert file paths from validation errors to SvelteKit routes + * + * Example paths: + * - data/bambu_lab/PLA/basic_pla/white/variant.json -> /Brand/bambu_lab/PLA/basic_pla/white + * - stores/amazon/store.json -> /Store/amazon + */ +export function pathToRoute(filePath: string): string { + // Normalize path separators + const normalizedPath = filePath.replace(/\\/g, '/'); + const parts = normalizedPath.split('/'); + + // Handle stores directory + if (normalizedPath.startsWith('stores/')) { + // stores/store_name/store.json -> /Store/store_name + if (parts.length >= 2) { + return `/Store/${parts[1]}`; + } + } + + // Handle data directory hierarchy + if (normalizedPath.startsWith('data/')) { + const [_, brandId, materialId, filamentId, variantId, filename] = parts; + + // Brand level (data/brandId/brand.json) + if (filename === 'brand.json' || parts.length === 3) { + return `/Brand/${brandId}`; + } + + // Material level (data/brandId/materialId/material.json) + if (filename === 'material.json' || parts.length === 4) { + return `/Brand/${brandId}/${materialId}`; + } + + // Filament level (data/brandId/materialId/filamentId/filament.json) + if (filename === 'filament.json' || parts.length === 5) { + return `/Brand/${brandId}/${materialId}/${filamentId}`; + } + + // Variant level (data/brandId/materialId/filamentId/variantId/variant.json or sizes.json) + if (filename === 'variant.json' || filename === 'sizes.json' || parts.length === 6) { + return `/Brand/${brandId}/${materialId}/${filamentId}/${variantId}`; + } + } + + // Fallback to home if path doesn't match expected structure + return '/'; +} diff --git a/webui/src/routes/+layout.svelte b/webui/src/routes/+layout.svelte index 9e5f3b0ef7..0a2eb1fdf2 100644 --- a/webui/src/routes/+layout.svelte +++ b/webui/src/routes/+layout.svelte @@ -1,6 +1,9 @@