diff --git a/.github/workflows/update_profiles.yaml b/.github/workflows/update_profiles.yaml index 0213e310ed..ae26aa7284 100644 --- a/.github/workflows/update_profiles.yaml +++ b/.github/workflows/update_profiles.yaml @@ -4,9 +4,14 @@ on: schedule: - cron: "0 0 * * *" workflow_dispatch: + push: + branches: + - main jobs: update_slicer_profiles: + # Only run on main branch, not on PRs + if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request' runs-on: ubuntu-latest permissions: contents: write diff --git a/README.md b/README.md index 0f9fbe65d8..1c372b4060 100644 --- a/README.md +++ b/README.md @@ -35,33 +35,52 @@ git clone https://github.com/YOUR_USERNAME/open-filament-database.git cd open-filament-database ``` ### 5. Make your changes! -Either use the web editor by simply running these commands or [following the guide](docs/webui.md), if you want to do it manually you can [use this one](docs/manual.md) +Use the web editor (recommended) or edit files manually: + +**Using the WebUI (Recommended):** ```bash cd webui npm ci npm run dev ``` -and access it in your browser at http://localhost:5173 +Then access it in your browser at http://localhost:5173 + +The WebUI includes built-in validation and data sorting features to help ensure your changes are correct. [Full WebUI guide](docs/webui.md) + +**Manual editing:** If you prefer to edit files directly, [follow this guide](docs/manual.md) + +### 6. Validate and sort your changes +The WebUI can validate and sort your data automatically: + +1. Click the "Validate" button in the top-right corner to check for errors +2. Click the "Sort Data" button to organize your JSON files consistently +3. Fix any validation errors that appear (they'll be highlighted in red) -### 6. Validate your changes -Once you've finished modifying the database you can use these commands or [this guide](docs/validation.md) to make sure your data is correct, fix any errors that pop up +Alternatively, you can use the command-line validation scripts ([see guide](docs/validation.md)): ```bash -python data_validator.py --folder-names # Validates folder names. -python data_validator.py --logo-files # Validates logo files. -python data_validator.py --json-files # Validates json files. -python data_validator.py --store-ids # Validates store ids. +python data_validator.py --folder-names # Validates folder names +python data_validator.py --logo-files # Validates logo files +python data_validator.py --json-files # Validates JSON files +python data_validator.py --store-ids # Validates store IDs ``` ### 7. Submit your changes -Start by running this command to add all your changes up +Before submitting, make sure your data is sorted consistently: +- **In the WebUI:** Click the "Sort Data" button in the top-right corner +- **Or via command line:** Run `python scripts/sort_data.py` + +Then add your changes: ```bash git add . ``` -Then run this command but replace `COMMIT_MESSAGE` with a title of what you did, e.g. "Added filament A to brand B" + +Create a commit with a descriptive message (e.g., "Added Elegoo Red PLA variant"): ```bash git commit -m "COMMIT_MESSAGE" ``` -When that's done you can run this command to upload your stuff + +Upload your changes to GitHub: ```bash git push -u origin YOUR_BRANCHNAME ``` -Afterwards you can make a pull request [using this guide](docs/pull-requesting.md) + +Finally, make a pull request [using this guide](docs/pull-requesting.md) diff --git a/data_validator.py b/data_validator.py index 8a76ae611e..259863976d 100644 --- a/data_validator.py +++ b/data_validator.py @@ -1,6 +1,7 @@ import json import re import os +import sys from concurrent.futures import ProcessPoolExecutor, as_completed from dataclasses import dataclass, field from enum import Enum @@ -49,6 +50,15 @@ def __str__(self) -> str: path_str = f" [{self.path}]" if self.path else "" return f"{self.level.value} - {self.category}: {self.message}{path_str}" + def to_dict(self) -> Dict[str, Any]: + """Convert to dictionary for JSON serialization.""" + return { + 'level': self.level.value, + 'category': self.category, + 'message': self.message, + 'path': str(self.path) if self.path else None + } + @dataclass class ValidationResult: @@ -73,6 +83,15 @@ def error_count(self) -> int: def warning_count(self) -> int: return len([e for e in self.errors if e.level == ValidationLevel.WARNING]) + def to_dict(self) -> Dict[str, Any]: + """Convert to dictionary for JSON serialization.""" + return { + 'errors': [e.to_dict() for e in self.errors], + 'error_count': self.error_count, + 'warning_count': self.warning_count, + 'is_valid': self.is_valid + } + # ------------------------- # Utility Functions @@ -781,11 +800,24 @@ class ValidationOrchestrator: def __init__(self, data_dir: Path = Path("./data"), stores_dir: Path = Path("./stores"), - max_workers: Optional[int] = None): + max_workers: Optional[int] = None, + progress_mode: bool = False): self.data_dir = data_dir self.stores_dir = stores_dir self.max_workers = max_workers self.schema_cache = SchemaCache() + self.progress_mode = progress_mode + + def emit_progress(self, stage: str, percent: int, message: str = '') -> None: + """Emit progress event as JSON to stdout for SSE streaming.""" + if self.progress_mode and hasattr(sys.stdout, 'isatty') and not sys.stdout.isatty(): + # Only emit when stdout is piped (not terminal) + print(json.dumps({ + 'type': 'progress', + 'stage': stage, + 'percent': percent, + 'message': message + }), flush=True) def run_tasks_parallel(self, tasks: List[ValidationTask]) -> ValidationResult: """Run validation tasks in parallel using process pool.""" @@ -814,34 +846,42 @@ def run_tasks_parallel(self, tasks: List[ValidationTask]) -> ValidationResult: def validate_json_files(self) -> ValidationResult: """Validate all JSON files against schemas.""" - print("Collecting JSON validation tasks...") + if not self.progress_mode: + print("Collecting JSON validation tasks...") tasks = collect_json_validation_tasks(self.data_dir, self.stores_dir) - print(f"Running {len(tasks)} JSON validation tasks...") + if not self.progress_mode: + print(f"Running {len(tasks)} JSON validation tasks...") return self.run_tasks_parallel(tasks) def validate_logo_files(self) -> ValidationResult: """Validate all logo files.""" - print("Collecting logo validation tasks...") + if not self.progress_mode: + print("Collecting logo validation tasks...") tasks = collect_logo_validation_tasks(self.data_dir, self.stores_dir) - print(f"Running {len(tasks)} logo validation tasks...") + if not self.progress_mode: + print(f"Running {len(tasks)} logo validation tasks...") return self.run_tasks_parallel(tasks) def validate_folder_names(self) -> ValidationResult: """Validate all folder names.""" - print("Collecting folder name validation tasks...") + if not self.progress_mode: + print("Collecting folder name validation tasks...") tasks = collect_folder_validation_tasks(self.data_dir, self.stores_dir) - print(f"Running {len(tasks)} folder name validation tasks...") + if not self.progress_mode: + print(f"Running {len(tasks)} folder name validation tasks...") return self.run_tasks_parallel(tasks) def validate_store_ids(self) -> ValidationResult: """Validate store IDs.""" - print("Validating store IDs...") + if not self.progress_mode: + print("Validating store IDs...") validator = StoreIdValidator(self.schema_cache) return validator.validate_store_ids(self.data_dir, self.stores_dir) def validate_gtin(self) -> ValidationResult: """Validate GTIN/EAN rules.""" - print("Validating GTIN/EAN...") + if not self.progress_mode: + print("Validating GTIN/EAN...") validator = GTINValidator(self.schema_cache) return validator.validate_gtin_ean(self.data_dir) @@ -850,15 +890,32 @@ def validate_all(self) -> ValidationResult: result = ValidationResult() # Check for missing files first - print("Checking for missing required files...") + self.emit_progress('missing_files', 0, 'Checking for missing required files...') + if not self.progress_mode: + print("Checking for missing required files...") validator = MissingFileValidator(self.schema_cache) result.merge(validator.validate_required_files(self.data_dir, self.stores_dir)) + self.emit_progress('missing_files', 100, 'Missing files check complete') + self.emit_progress('json_files', 0, 'Validating JSON files...') result.merge(self.validate_json_files()) + self.emit_progress('json_files', 100, 'JSON validation complete') + + self.emit_progress('logo_files', 0, 'Validating logo files...') result.merge(self.validate_logo_files()) + self.emit_progress('logo_files', 100, 'Logo validation complete') + + self.emit_progress('folder_names', 0, 'Validating folder names...') result.merge(self.validate_folder_names()) + self.emit_progress('folder_names', 100, 'Folder name validation complete') + + self.emit_progress('store_ids', 0, 'Validating store IDs...') result.merge(self.validate_store_ids()) + self.emit_progress('store_ids', 100, 'Store ID validation complete') + + self.emit_progress('gtin', 0, 'Validating GTIN/EAN...') result.merge(self.validate_gtin()) + self.emit_progress('gtin', 100, 'GTIN/EAN validation complete') return result @@ -876,15 +933,18 @@ def main(): parser.add_argument("--folder-names", action="store_true", help="Validate folder names") parser.add_argument("--store-ids", action="store_true", help="Validate store IDs") + parser.add_argument("--json", action="store_true", help="Output results as JSON") + parser.add_argument("--progress", action="store_true", help="Emit progress events (for SSE streaming)") args = parser.parse_args() - orchestrator = ValidationOrchestrator(max_workers=os.cpu_count()) + orchestrator = ValidationOrchestrator(max_workers=os.cpu_count(), progress_mode=args.progress) result = ValidationResult() # Run requested validations - if not any(vars(args).values()): - print("No args passed, validating all") + if not any([args.json_files, args.logo_files, args.folder_names, args.store_ids]): + if not args.json and not args.progress: + print("No args passed, validating all") result = orchestrator.validate_all() else: if args.json_files: @@ -896,28 +956,39 @@ def main(): if args.store_ids: result.merge(orchestrator.validate_store_ids()) - # Print results - if result.errors: - # Group errors by category - errors_by_category: Dict[str, List[ValidationError]] = {} - for error in result.errors: - if error.category not in errors_by_category: - errors_by_category[error.category] = [] - errors_by_category[error.category].append(error) - - # Print errors grouped by category - for category, errors in sorted(errors_by_category.items()): - print(f"\n{category} ({len(errors)}):") - print("-" * 80) - for error in errors: - print(f" {error}") - - print( - f"\nValidation failed: {result.error_count} errors, {result.warning_count} warnings") - exit(1) + # Output results + if args.json: + # JSON output mode + output = result.to_dict() + # Use compact output in progress mode for SSE compatibility, pretty output otherwise + if args.progress: + print(json.dumps(output)) + else: + print(json.dumps(output, indent=2)) + sys.exit(0 if result.is_valid else 1) else: - print("All validations passed!") - exit(0) + # Text output mode + if result.errors: + # Group errors by category + errors_by_category: Dict[str, List[ValidationError]] = {} + for error in result.errors: + if error.category not in errors_by_category: + errors_by_category[error.category] = [] + errors_by_category[error.category].append(error) + + # Print errors grouped by category + for category, errors in sorted(errors_by_category.items()): + print(f"\n{category} ({len(errors)}):") + print("-" * 80) + for error in errors: + print(f" {error}") + + print( + f"\nValidation failed: {result.error_count} errors, {result.warning_count} warnings") + exit(1) + else: + print("All validations passed!") + exit(0) if __name__ == '__main__': diff --git a/docs/cloning.md b/docs/cloning.md index 7a23b81838..20096b9aa1 100644 --- a/docs/cloning.md +++ b/docs/cloning.md @@ -1,16 +1,19 @@ -# Cloning your repository -So now we need to clone our repository, that may sound like we're gonna do like in Star Wars and create an army but sadly it's just the process of getting your cloud folder onto your computer, you can imagine this as copy pasting a folder from the cloud. +# Cloning Your Repository +Now we need to clone your repository. "Cloning" is the process of downloading your forked copy from GitHub to your computer - think of it as copying a folder from the cloud to your local machine. -To be able to make a PR we also need to make what's called a branch after cloning, we won't be explaining this as it's require a greater understanding of git though if you're studious [here's a link](https://www.w3schools.com/git/git_branch.asp) +After cloning, we'll create a "branch" for your changes. Branches allow you to work on changes without affecting the main codebase. If you're curious about how branches work, [here's a helpful guide](https://www.w3schools.com/git/git_branch.asp). ## Cloning on Windows -To clone your files you simply need to open a commandline window, we'll do this by holding the `windows key` and pressing `R`, this will spawn a small little windowin the bottom left with a text input, simply type in `cmd` and press `enter`. -Now, don't be scared by the hackerman window that appeared. -You now have to write or copy in the following command while replacing `YOUR_USERNAME` with you username on github +To clone your files, you need to open a command line window: +1. Hold the `Windows key` and press `R` +2. A small window will appear in the bottom left with a text input +3. Type `cmd` and press `Enter` + +A black terminal window will appear. Now type or paste the following command, replacing `YOUR_USERNAME` with your GitHub username: ```bash git clone https://github.com/YOUR_USERNAME/open-filament-database.git ``` -press enter and let it run, when it allows you to write again you then enter the following two lines, first one will be speedy. +Press `Enter` and let it run. When it finishes and you can type again, enter the following two commands: **Remember to replace YOUR_BRANCHNAME with a descriptive name for your changes.** Use lowercase with hyphens, for example: @@ -25,8 +28,8 @@ git checkout -b YOUR_BRANCHNAME Leave the window open and continue with [Step 5 in the README](../README.md#5-make-your-changes) to make your changes. -## Cloning on Linux and MacOS -Simply open you terminal and run the following to clone your repository and create a branch for your changes +## Cloning on Linux and macOS +Open your terminal and run the following commands to clone your repository and create a branch for your changes: ```bash git clone https://github.com/YOUR_USERNAME/open-filament-database.git cd open-filament-database diff --git a/docs/forking.md b/docs/forking.md index fcc64bc6d6..e277cacb9d 100644 --- a/docs/forking.md +++ b/docs/forking.md @@ -1,7 +1,13 @@ -# To fork a repository -So to fork our repository you'll need to open it in your web browser, you can then find the "Fork" button in the top right of the page, on the other side of the name below the search -![Fork button getting pressed](img/forking01.png) -You'll get redirected to a menu where you can fork the repository, if the owner menu says "Choose an owner" please click it and select yourself, then click "Create fork" -![Create a new Fork menu](img/forking02.png) +# Forking the Repository +To create your own copy of the repository, you need to "fork" it on GitHub: -You should now be looking at the same page again except your name has replaced SimplyPrint (or OpenFilamentCollective) in the top left! \ No newline at end of file +1. Open the [Open Filament Database repository](https://github.com/OpenFilamentCollective/open-filament-database) in your web browser +2. Find the "Fork" button in the top right of the page (below the search bar) + ![Fork button getting pressed](img/forking01.png) + +3. You'll be redirected to a menu where you can create your fork +4. If the owner menu says "Choose an owner", click it and select your username +5. Click "Create fork" + ![Create a new Fork menu](img/forking02.png) + +You should now see the same repository page, but with your username in the top left instead of OpenFilamentCollective. This is your personal copy that you can modify freely! \ No newline at end of file diff --git a/docs/installing-software.md b/docs/installing-software.md index 3d4e673457..265d57f658 100644 --- a/docs/installing-software.md +++ b/docs/installing-software.md @@ -1,5 +1,5 @@ -# What to install and how to do it -In this document we'll go through what we need to install and how to do it, depending on your computer. +# Software Installation Guide +This guide will walk you through installing the required software for contributing to the database. - [Git](#git) - [Python](#python) @@ -7,7 +7,7 @@ In this document we'll go through what we need to install and how to do it, depe - [NodeJS/NPM](#nodejsnpm) ## Git -You'll need git installed to be able to share your changes with us, please follow the instructions below depending on your operating system (If you're in doubt you're properly on Windows) +Git is required to download the database and upload your changes. Follow the instructions for your operating system below.
Windows @@ -18,10 +18,10 @@ Go to https://git-scm.com/downloads and click the `Download for Windows` Button You'll most likely want to click on `Git for Windows/x64 Setup` on most systems as of writing ![](img/windowsGitDownload02.png) -Once the download is complete, click on the executable in the top right to start installing Git. Simply click through the setup wizard leaving all options on default. +Once the download is complete, click on the installer to start installing Git. Click through the setup wizard, leaving all options on their default settings. ![](img/windowsGitInstaller01.png) -Once it's done installing we'd recommend you uncheck `View Release Notes` and click finish, this will close the installer without opening a text file with info about git +When installation is complete, uncheck "View Release Notes" and click "Finish" to close the installer. ![](img/windowsGitInstaller02.png)
@@ -57,7 +57,7 @@ If you don't have homebrew you can also use the [latest macOS Git Installer](htt ## Python -You'll need python to run our data validator, please follow the instructions below depending on your operating system (If you're in doubt you're properly on Windows) +Python is required to run the data validation and sorting scripts. Follow the instructions for your operating system below.
Windows @@ -153,8 +153,8 @@ pip3 install -r requirements.txt > **Note:** You'll need to activate the virtual environment each time you open a new terminal to run the validator. -## NodeJS/NPM -You'll need NodeJS to be able to run our WebUI for easy editing of the data, please follow the instructions below depending on your operating system (If you're in doubt you're properly on Windows) +## Node.js/NPM +Node.js is required to run the WebUI for easy editing of the database. Follow the instructions for your operating system below.
Windows diff --git a/docs/manual.md b/docs/manual.md index 6a4a82f7eb..fa2de30016 100644 --- a/docs/manual.md +++ b/docs/manual.md @@ -1,5 +1,7 @@ -# Manually contributing to the database! -Below is much of the important information regarding how to contribute, in general we reommend reading through this document and then taking a look in the `/data` folder for reference \:D +# Manual Contribution Guide +This guide explains how to manually edit the database files. We recommend reading through this document first, then exploring the `/data` folder for reference examples. + +**Note:** Most contributors find the [WebUI](webui.md) easier to use than manual editing. Consider using the WebUI unless you have a specific reason to edit files directly. ## ๐Ÿ“ Project Structure The database is organized as a structured JSON-based hierarchy inside the `/data` directory, following this pattern: @@ -17,93 +19,99 @@ data/ โ””โ”€โ”€ variant.json ``` ## ๐Ÿงพ General Guidelines -- Each **brand** has it's own folder under `data/` which contains: - - A brand.json file that contains data about the brand - - The brands logo -- Each **material** type (e.g., PLA, PETG, ABS) has it's own subfolder inside the **brand** folder which contains a material.json file about it. -- Each **filament** (e.g. Bambu Lab's Basic Gradient) has it's own subfolder, it contains a filament.json file about it. -- Each **variant** of the **filament** (e.g. colours) has it's own subfolder with a sizes.json and variant.json files +- Each **brand** has its own folder under `data/` which contains: + - A `brand.json` file with brand information + - The brand's logo image +- Each **material** type (e.g., PLA, PETG, ABS) has its own subfolder inside the brand folder containing a `material.json` file +- Each **filament** (e.g., Bambu Lab's Basic Gradient) has its own subfolder containing a `filament.json` file +- Each **variant** of a filament (e.g., colors like Red, Blue, Black) has its own subfolder with `sizes.json` and `variant.json` files ### ๐Ÿท๏ธ Adding a Brand -- Go to the `data/` directory and create a new folder for your brand. -- Add the logo of your brand, it should: - - Max be 400x400, unless it's an svg. - - Named with lowercase snakecase. - - Be simple (e.g. "colorfab.png") -- Create a `brand.json` for your brand. The file should include: - - Brand name, using the `brand` key - - Website URL, using the `website` key - - The name of the logo file, using the `logo` key - - Country of origin (or an empty string), using the `origin` key +1. Go to the `data/` directory and create a new folder for your brand +2. Add the brand's logo: + - Maximum size: 400x400 pixels (SVG files can be any size) + - Naming: Use lowercase snake_case (e.g., `colorfab.png`) + - Keep the filename simple +3. Create a `brand.json` file with the following fields: + - `brand` - The brand name + - `website` - The brand's website URL + - `logo` - The filename of the logo (e.g., `colorfab.png`) + - `origin` - Country of origin (use an empty string `""` if unknown) ### ๐Ÿงช Adding a Material Type -- Go to your brands folder and create a new folder named after your material type. -- Create a `material.json` file. It should include: - - The material name (e.g., PLA, PETG), using the `material` - - Optionally you can also include the following keys: - - The default max dry temperature - - The default slicer settings, refer to the schema for info about this in `schemas/material_schema.json`. +1. Navigate to your brand's folder and create a new folder named after the material type +2. Create a `material.json` file with: + - `material` - The material name (e.g., `"PLA"`, `"PETG"`, `"ABS"`) + - Optional fields: + - Default maximum dry temperature + - Default slicer settings (refer to `schemas/material_schema.json` for details) ### ๐Ÿ“ฆ Adding a Filament -Each filament represents a product line (e.g., "Silk PLA", "Tough PLA", etc.), **not a specific color**. +Each filament represents a product line (e.g., "Silk PLA", "Tough PLA"), **not a specific color**. -- Go to your material type folder and create a new folder named after your filament. -- Create a `filament.json` file, It should include: - - The filaments name - - The diameter tolerance in mm - - The density of the filament - - Optionally you can also include the following keys: - - The max dry temperature - - A URL to the data sheet - - A URL to the safety sheet - - Whether it is discontinued or not - - The slicer ids and slicer settings, you should refer to the schema for info about this in `schemas/filament_schema.json`. +1. Navigate to your material type folder and create a new folder named after the filament +2. Create a `filament.json` file with: + - Required fields: + - Filament name + - Diameter tolerance (in mm) + - Filament density + - Optional fields: + - Maximum dry temperature + - Data sheet URL + - Safety sheet URL + - Discontinued status (boolean) + - Slicer IDs and settings (refer to `schemas/filament_schema.json` for details) ### ๐ŸŽจ Adding a Variant -- Go to your filament folder and create a new folder named after your variant, it includes the following two files. -#### Variant.json -- Create a `variant.json` file, It should include: - - A color name, in reality this is just your variant name but most variants are colours. - - A color hex, this hex code defines the colour of your variant. - - Optionally you can also include one or more of the following but we suggest you refer to the schema at `schemas/variant_schema.json` for more detail if you need to define these: - - Whether it is discontinued or not. - - A hex variants array which defines alternative color codes. - - The color standards of the variant. - - The traits of this variant. +Navigate to your filament folder and create a new folder named after the variant. Create the following two files: + +#### variant.json +Create a `variant.json` file with: +- Required fields: + - `color_name` - The variant name (usually a color like "Red" or "Black") + - `color_hex` - Hex color code representing the variant (e.g., `"#FF0000"`) +- Optional fields (see `schemas/variant_schema.json` for details): + - `discontinued` - Whether the variant is discontinued (boolean) + - `hex_variants` - Array of alternative hex color codes + - `color_standards` - Standard color codes (RAL, Pantone, etc.) + - `traits` - Special properties (e.g., "glow-in-the-dark", "silk") + +#### sizes.json +Create a `sizes.json` file containing an array of size objects. Each object includes: +- Required fields: + - Filament weight (in grams) + - Filament diameter (in mm, typically `1.75` or `2.85`) +- Optional fields: + - Empty spool weight + - Spool core diameter + - EAN code + - Internal article number + - Barcode/NFC/QR identifiers + - Discontinued status + - `purchase_links` - Array of purchase links (highly recommended): + - `store_id` - Reference to a store in the `/stores` directory + - `url` - Link to the product page + - `is_affiliate` - Whether this is an affiliate link (boolean) -#### Sizes.json -- Create a `sizes.json` file, It should include an array of objects that each define the following: - - The variants filament weight. - - The variants diameter. - - Optionally you can also include the following keys: - - The empty spool weight. - - The spool core diameter. - - The ean of this variant. - - The internal article number. - - The barcode identifier. - - The nfc identifier. - - The qr identifier. - - Whether or not it is discontinued. - - An array of purchase links, this is highly recommended and mostly just includes the following few traits, for more detail please refer to the schema in `schemas/sizes_schema.json`. - - A store id, mostly this'll be a string that refers to a store inside `/stores` directory. - - A url to the shop page, preferably this'll be the exact variant but the general filament page works in a pinch. - - Whether or not it is an affiliate link. +For detailed schema information, see `schemas/sizes_schema.json`. ### ๐Ÿช Adding a Store -Stores are referenced in purchase links and live in the `/stores` directory. +Stores are referenced in purchase links and are stored in the `/stores` directory. -- Create a new folder in `/stores` named after your store (use lowercase with underscores for the folder name, e.g., `my_store`). -- Add the store logo, it should: - - Max be 400x400, unless it's an svg. - - Named with lowercase snakecase (e.g. "my_store.png"). -- Create a `store.json` file. It should include: - - `id` - The store identifier (should match the folder name) - - `name` - The display name of the store - - `storefront_url` - The URL to the store's homepage - - `logo` - The filename of the logo image - - `ships_from` - An array of locations the store ships from (can be empty) - - `ships_to` - An array of locations the store ships to (can be empty) - - Optionally: `storefront_affiliate_link` - An affiliate link to the storefront +1. Create a new folder in `/stores` using lowercase snake_case (e.g., `amazon_us`, `printed_solid`) +2. Add the store logo: + - Maximum size: 400x400 pixels (SVG files can be any size) + - Naming: Use lowercase snake_case matching the folder name (e.g., `amazon_us.png`) +3. Create a `store.json` file with: + - Required fields: + - `id` - Store identifier (must match the folder name) + - `name` - Display name of the store + - `storefront_url` - URL to the store's homepage + - `logo` - Filename of the logo image + - `ships_from` - Array of shipping origin locations (use `[]` if unknown) + - `ships_to` - Array of shipping destination locations (use `[]` if unknown) + - Optional fields: + - `storefront_affiliate_link` - Affiliate link to the storefront -For more details, refer to the schema in `schemas/store_schema.json`. +For detailed schema information, see `schemas/store_schema.json`. diff --git a/docs/pull-requesting.md b/docs/pull-requesting.md index a2e34b540f..f9c49fb755 100644 --- a/docs/pull-requesting.md +++ b/docs/pull-requesting.md @@ -1,34 +1,58 @@ -# Pull requesting -To make a pull request we assume you have your data in the cloud already so let's get started! - -To begin with we'll go to this link, the pull requests tab of the main database -https://github.com/OpenFilamentCollective/open-filament-database/pulls -If you've pushed a yellowish banner will appear, click the compare and pull request button. -![](img/pullRequesting01.png) -You should be brought to a page that looks like this, simply change the title to whatever describes what you've changed and write a short description about the same -![](img/pullRequesting02.png) -When you're ready press the `Create pull request` button and wait for someone to come merge it, or be ready for some feedback \:D - -### Alternative method -If the last method didn't work, because of the yellow banner not appearing, it is also possible to go to your profile, and go to your repositories \ -![](img/pullRequesting03.png) \ -After getting to your repository list you can search up your version of the database \ -![](img/pullRequesting04.png) \ -Click on the database and then the pull requests tab \ -![](img/pullRequesting05.png) \ -You can then click the "New Pull request" button \ -![](img/pullRequesting06.png) -Find your branch on the right side \ -![](img/pullRequesting07.png) -Click the "Create pull request" button -![](img/pullRequesting08.png) -And then you just fill it out with whatever info you think we need and wait for someone to come merge it, or be ready for some feedback \:D -![](img/pullRequesting02.png) - -### Using PR Templates +# Creating a Pull Request +Before creating a pull request, make sure you have: +1. Validated your changes (no errors should remain) +2. Sorted your data using the WebUI or `sort_data.py` script +3. Pushed your changes to GitHub + +## Method 1: Quick Pull Request + +1. Go to the [pull requests tab](https://github.com/OpenFilamentCollective/open-filament-database/pulls) of the main database +2. If you've recently pushed changes, a yellow banner will appear +3. Click the "Compare & pull request" button + ![](img/pullRequesting01.png) + +4. You'll be brought to a page to create your pull request: + - Change the title to describe what you've changed (e.g., "Add Elegoo Red PLA variant") + - Write a short description explaining your changes + ![](img/pullRequesting02.png) + +5. Click "Create pull request" +6. A maintainer will review your changes and either merge them or provide feedback + +## Method 2: Alternative Method +If the yellow banner doesn't appear, follow these steps: + +1. Go to your GitHub profile and click on "Repositories" + ![](img/pullRequesting03.png) + +2. Search for your forked version of the database + ![](img/pullRequesting04.png) + +3. Click on the database, then click the "Pull requests" tab + ![](img/pullRequesting05.png) + +4. Click the "New Pull request" button + ![](img/pullRequesting06.png) + +5. Find and select your branch on the right side + ![](img/pullRequesting07.png) + +6. Click "Create pull request" + ![](img/pullRequesting08.png) + +7. Fill in the title and description, then click "Create pull request" + ![](img/pullRequesting02.png) + +8. Wait for a maintainer to review and merge your changes, or be ready to address any feedback! + +## Using Pull Request Templates When creating a pull request, you can use one of our templates to help structure your submission: -- **Data Addition** - Use this when adding new brands, materials, filaments, variants, or stores. It includes checklists for validation and data quality. -- **WebUI Changes** - Use this when making changes to the web interface code. +- **Data Addition** - Use when adding new brands, materials, filaments, variants, or stores + - Includes checklists for validation and data quality + - Ensures you've sorted your data before submitting + +- **WebUI Changes** - Use when making changes to the web interface code + - For developers modifying the WebUI application -To use a template, look for the "Choose a template" option when creating your PR, or you can find them in the `.github/PULL_REQUEST_TEMPLATE/` folder. \ No newline at end of file +To use a template, look for the "Choose a template" option when creating your PR, or find them in the `.github/PULL_REQUEST_TEMPLATE/` folder. \ No newline at end of file diff --git a/docs/validation.md b/docs/validation.md index 1c537ac08e..96656af1ec 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -1,30 +1,80 @@ -# Validation of your files -To make sure all your files are correct we need to run some code on them to check, for this you need python which if you haven't installed yet [can be found how to here](installing-software.md#python). -After you've got python installed you can go back to the open-filament-database folder, assuming just used the WebUI, by writting +# Data Validation +To ensure all your files are correct, you can validate them using either the WebUI (recommended) or the command-line Python scripts. + +## Option 1: Validate Using the WebUI (Recommended) +The easiest way to validate your changes is directly in the WebUI: + +1. While editing in the WebUI, validation runs automatically in the background +2. Look for the "Validation" dropdown in the top-right corner +3. Click the "Validate" button to run a full validation check +4. Any errors or warnings will appear in the dropdown with links to the problematic data +5. Click on an error to navigate directly to the issue + +This method is recommended because it provides immediate feedback and makes it easy to locate and fix issues. + +## Option 2: Validate Using Python Scripts +If you prefer to validate using the command line, you'll need Python installed. If you haven't installed Python yet, [follow this guide](installing-software.md#python). + +After using the WebUI, navigate back to the open-filament-database folder: ```bash cd .. ``` -And then running the following depending on platform, if any text appears that between pressing enter and being able to input another command read through it and see if you can fix it. + +Then run the validation scripts based on your platform. If any errors appear, read through them carefully and fix the issues. ### Windows ```bash -python.exe data_validator.py --folder-names # Validates folder names. -python.exe data_validator.py --logo-files # Validates logo files. -python.exe data_validator.py --json-files # Validates json files. -python.exe data_validator.py --store-ids # Validates store ids. +python.exe data_validator.py --folder-names # Validates folder names +python.exe data_validator.py --logo-files # Validates logo files +python.exe data_validator.py --json-files # Validates JSON files +python.exe data_validator.py --store-ids # Validates store IDs ``` -### Linux/macos +### Linux/macOS ```bash -python data_validator.py --folder-names # Validates folder names. -python data_validator.py --logo-files # Validates logo files. -python data_validator.py --json-files # Validates json files. -python data_validator.py --store-ids # Validates store ids. +python data_validator.py --folder-names # Validates folder names +python data_validator.py --logo-files # Validates logo files +python data_validator.py --json-files # Validates JSON files +python data_validator.py --store-ids # Validates store IDs ``` -if this gives an error about `python` not being installed try replacing every instance of `python` with `python3` + +If this gives an error about `python` not being installed, try replacing `python` with `python3`: ```bash -python3 data_validator.py --folder-names # Validates folder names. -python3 data_validator.py --logo-files # Validates logo files. -python3 data_validator.py --json-files # Validates json files. -python3 data_validator.py --store-ids # Validates store ids. +python3 data_validator.py --folder-names # Validates folder names +python3 data_validator.py --logo-files # Validates logo files +python3 data_validator.py --json-files # Validates JSON files +python3 data_validator.py --store-ids # Validates store IDs ``` + +## Understanding Validation Results +- **Errors** (shown in red) are critical issues that must be fixed before submitting your pull request +- **Warnings** (shown in yellow) are suggestions for improvement but won't block your contribution +- Each validation message includes the file path and specific issue to help you locate and fix problems quickly + +## Sorting Your Data +Before submitting your changes, you should sort all JSON files to ensure consistency across the database. This makes it easier to review changes and maintain the codebase. + +### Using the WebUI (Recommended) +1. Click the "Sort Data" button in the top-right corner of the WebUI +2. Wait for the sorting process to complete +3. The progress will be shown in a modal window + +### Using Python Scripts +Navigate to the open-filament-database folder and run: + +**Windows:** +```bash +python.exe scripts/sort_data.py +``` + +**Linux/macOS:** +```bash +python scripts/sort_data.py +``` + +Or if you need to use `python3`: +```bash +python3 scripts/sort_data.py +``` + +The sorting script will organize all JSON files alphabetically and format them consistently. This is an important step before creating your pull request. diff --git a/docs/webui.md b/docs/webui.md index e89c797d46..f3d128c30d 100644 --- a/docs/webui.md +++ b/docs/webui.md @@ -1,25 +1,58 @@ -# WebUI, User interface but on the web -The WebUI is a simple but effective user interface which allows you to modify the database in a nice and easy way without having to crawl through terminals or files. +# WebUI - User Interface on the Web +The WebUI is a simple but effective user interface that allows you to modify the database in a nice and easy way without having to crawl through terminals or files. -The WebUI is structured in layers, like shown below +## Structure +The WebUI is structured in layers, as shown below: ``` -[brand (Bambu lab, ESun3D, SUNLU)] +[brand (Bambu Lab, eSUN, SUNLU)] โ””โ”€โ”€ [material (e.g. PLA, ABS, PETG)] โ””โ”€โ”€ [filament (e.g. sparkly, pla basic)] โ””โ”€โ”€ [variant (e.g. Black, Rainbow, Variant B)] ``` -So if you want to add a brand you can add that on the homepage of the website, then you can modify the brands materials, filaments and the individual variants of those filaments. +You can add a brand on the homepage of the website, then modify the brand's materials, filaments, and the individual variants of those filaments. -To run the WebUI simply take your previous terminal window and run the following lines in it -1. Enter the WebUI folder and install the WebUI needs automatically +## Getting Started + +1. Enter the WebUI folder and install dependencies: ```bash cd webui npm ci ``` -2. Start the website +2. Start the development server: ```bash npm run dev ``` -3. Navigate to the link printed in the terminal, most of the time this will be http://localhost:5173 but it might vary slightly. -4. Modify to your hearts content! \ No newline at end of file + +3. Navigate to the link printed in the terminal, typically http://localhost:5173 +4. Start editing the database! + +## Features + +### Data Validation +The WebUI includes built-in validation to help ensure your data is correct before submitting. You can validate your changes in two ways: + +1. **Automatic Validation** - The WebUI automatically checks your data as you make changes and displays any issues in the navigation bar +2. **Manual Validation** - Click the "Validate" button in the top-right corner to run a full validation check + +When validation issues are found: +- A badge appears on the "Validation" dropdown showing the number of errors and warnings +- Errors are displayed in red (critical issues that must be fixed) +- Warnings are displayed in yellow (suggestions for improvement) +- Click on any validation issue to jump directly to the problematic data + +### Data Sorting +The WebUI can automatically sort your data files to ensure consistency across the database: + +1. Click the "Sort Data" button in the top-right corner +2. The sorting process will run and organize all JSON files alphabetically +3. A progress modal will show you the sorting status + +This ensures that all contributors follow the same data organization, making it easier to review changes and maintain the database. + +### Real-time Progress +When running validation or sorting operations, you'll see a progress modal that shows: +- Current stage of the operation +- Progress percentage +- Detailed status messages +- Any errors that occur during the process \ No newline at end of file diff --git a/scripts/migrate_schema.py b/scripts/migrate_schema.py deleted file mode 100644 index dc9ef208ed..0000000000 --- a/scripts/migrate_schema.py +++ /dev/null @@ -1,1229 +0,0 @@ -#!/usr/bin/env python3 -""" -migrate_schema.py - Migrate data files to new schema format - -This script transforms the existing data files to conform to the updated schemas: -- brand.json: Rename 'brand' -> 'name', add 'id', rename logo files, rename folder -- filament.json: Add 'id' field, rename folder -- variant.json: Rename 'color_name' -> 'name', add 'id', rename folder -- store.json: Rename logo files, rename folder -- sizes.json: Update store_id references, migrate spool_refill to size level -- material.json: Map material types to valid enum values from material_types_schema.json - -The script processes stores first to discover ID mappings (old_id -> new_id), -then uses these mappings to automatically update store_id references in data files. - -For spool_refill migration: The new schema moves spool_refill from purchase_link -level to size level. If a size has mixed refill/non-refill purchase links, they -are split into separate size entries. - -For material merging: When a material type maps to an existing folder (e.g., -"Carbon Fibre" -> "PETG"), filaments are merged. Unique filaments are moved, -duplicate filaments have their data merged (missing variants/fields copied), -then the source material folder is deleted. - -Usage: - python scripts/migrate_schema.py --dry-run # Preview changes - python scripts/migrate_schema.py # Apply changes -""" - -import argparse -import json -import os -import re -import shutil -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - - -# Mapping from old material names to schema enum values -# See schemas/material_types_schema.json for valid enum values -MATERIAL_TYPE_MAP: dict[str, str] = { - # Nylon variants -> PA types - "Nylon": "PA12", - "PAHT": "PA12", - "PA": "PA6", - "PA-CF": "PA6", - "PA-GF": "PA6", - # PETG variants - "PETG-CF": "PETG", - "Addnite": "PETG", - "Carbon Fibre": "PETG", # Add-North Carbon Fibre is PETG-based - "Graphene": "PETG", # Add-North Graphene is PETG-based - # PET variants - "PE": "PET", - "PET-CF": "PET", - # PLA variants - "APLA": "PLA", - "Bio-performance": "PLA", - "AquaPrint": "PLA", - "Woodfill": "PLA", - "Special": "PLA", - # PPA variants - "PPA-CF": "PPA", - # CPE variants - "CPE-HT": "CPE", - # PP variants - "CPX-PP": "PP", - # PC variants - "CPX-KyronMax": "PC", - # EVA variants - "EVA+": "EVA", - # Support materials (typically PVA-based) - "Support": "PVA", - # Composite materials - "Boron Carbide": "PA12", -} - - -@dataclass -class MigrationIssue: - """Represents an issue encountered during migration.""" - issue_type: str - path: str - message: str - - -@dataclass -class MigrationReport: - """Collects issues and warnings during migration.""" - issues: list[MigrationIssue] = field(default_factory=list) - - def add_issue(self, issue_type: str, path: str, message: str) -> None: - self.issues.append(MigrationIssue(issue_type, path, message)) - - def has_issues(self) -> bool: - return len(self.issues) > 0 - - def print_summary(self) -> None: - if not self.issues: - print("\nNo issues encountered during migration.") - return - - print(f"\n{'=' * 60}") - print(f"MIGRATION ISSUES ({len(self.issues)} total)") - print('=' * 60) - - # Group by issue type - by_type: dict[str, list[MigrationIssue]] = {} - for issue in self.issues: - by_type.setdefault(issue.issue_type, []).append(issue) - - for issue_type, issues in sorted(by_type.items()): - print(f"\n{issue_type} ({len(issues)}):") - for issue in issues: - print(f" - {issue.path}") - print(f" {issue.message}") - - -def generate_id(name: str) -> str: - """Convert a name to a valid id (lowercase, spaces/hyphens/ampersands to underscores). - - Examples: - "Bambu Lab" -> "bambu_lab" - "3DO" -> "3do" - "Add-North" -> "add_north" - "ABS+" -> "abs+" - "Black&White" -> "black_white" - """ - id_str = name.lower() - id_str = re.sub(r'[\s\-&]+', '_', id_str) # spaces/hyphens/ampersands -> underscore - id_str = re.sub(r'[^a-z0-9_+]', '', id_str) # remove invalid chars (keep +) - id_str = re.sub(r'_+', '_', id_str) # collapse multiple underscores - return id_str.strip('_') - - -def rename_folder(folder: Path, new_name: str, dry_run: bool) -> Path: - """Rename a folder to a new name. - - Handles case-only renames on case-insensitive filesystems by using - a two-step rename through a temporary name. - - Returns: - The new path after renaming (or the original if no rename needed). - """ - if folder.name == new_name: - return folder - - new_path = folder.parent / new_name - if new_path.exists(): - # Check if it's the same folder (case-insensitive rename like Black-White -> black_white) - try: - if folder.samefile(new_path): - # Same folder, different case - do a two-step rename via temp name - if dry_run: - print(f" Would rename folder: {folder.name} -> {new_name}") - else: - temp_path = folder.parent / f".{new_name}_temp_{os.getpid()}" - shutil.move(str(folder), str(temp_path)) - shutil.move(str(temp_path), str(new_path)) - print(f" Renamed folder: {folder.name} -> {new_name}") - return new_path - except (OSError, ValueError): - pass - - print(f" Warning: Cannot rename {folder.name} -> {new_name}, target exists") - return folder - - if dry_run: - print(f" Would rename folder: {folder.name} -> {new_name}") - else: - shutil.move(str(folder), str(new_path)) - print(f" Renamed folder: {folder.name} -> {new_name}") - - return new_path - - -def find_logo_file(directory: Path) -> Path | None: - """Find a logo file in the given directory.""" - logo_extensions = ['.png', '.jpg', '.jpeg', '.svg'] - for f in directory.iterdir(): - if f.is_file() and f.suffix.lower() in logo_extensions: - return f - return None - - -def rename_logo(directory: Path, current_logo: str, dry_run: bool) -> str | None: - """Rename logo file to logo.{ext} and return new name. - - Returns: - New logo filename (e.g., "logo.png") or None if no logo found. - """ - if not current_logo: - return None - - # Already in correct format - if re.match(r'^logo\.(png|jpg|svg)$', current_logo): - return current_logo - - # Find the current logo file - current_path = directory / current_logo - if not current_path.exists(): - # Try to find any logo file - logo_file = find_logo_file(directory) - if logo_file: - current_path = logo_file - else: - print(f" Warning: Logo file not found: {current_logo} in {directory}") - return None - - # Determine new name - ext = current_path.suffix.lower() - if ext == '.jpeg': - ext = '.jpg' - new_name = f"logo{ext}" - new_path = directory / new_name - - if current_path != new_path: - if dry_run: - print(f" Would rename logo: {current_path.name} -> {new_name}") - else: - shutil.move(str(current_path), str(new_path)) - print(f" Renamed logo: {current_path.name} -> {new_name}") - - return new_name - - -def load_json(path: Path) -> dict[str, Any] | None: - """Load a JSON file, returning None if it doesn't exist.""" - if not path.exists(): - return None - with open(path, 'r', encoding='utf-8') as f: - return json.load(f) - - -def save_json(path: Path, data: dict[str, Any], dry_run: bool) -> None: - """Save data to a JSON file.""" - if dry_run: - return - with open(path, 'w', encoding='utf-8') as f: - json.dump(data, f, indent=4, ensure_ascii=False) - f.write('\n') - - -def migrate_spool_refill(sizes_data: list[dict]) -> tuple[list[dict], bool]: - """Migrate spool_refill from purchase_link level to size level. - - Per the new schema, spool_refill is now on the size object and applies to all - purchase_links within that size. If a size has mixed refill/non-refill links, - we split them into separate size entries. - - Args: - sizes_data: List of size objects from sizes.json. - - Returns: - Tuple of (updated sizes list, whether changes were made). - """ - new_sizes = [] - changed = False - - for size_obj in sizes_data: - if not isinstance(size_obj, dict): - new_sizes.append(size_obj) - continue - - purchase_links = size_obj.get('purchase_links', []) - if not purchase_links: - new_sizes.append(size_obj) - continue - - # Separate links by spool_refill status - refill_links = [] - non_refill_links = [] - - for link in purchase_links: - if not isinstance(link, dict): - non_refill_links.append(link) - continue - - is_refill = link.get('spool_refill', False) - # Remove spool_refill from the link (it's deprecated at this level) - link_copy = {k: v for k, v in link.items() if k != 'spool_refill'} - - if is_refill: - refill_links.append(link_copy) - changed = True # We're removing/moving spool_refill - else: - # Only mark changed if we actually removed a spool_refill key - if 'spool_refill' in link: - changed = True - non_refill_links.append(link_copy) - - # Create size entries based on what we found - if non_refill_links: - # Non-refill size (original size without spool_refill) - non_refill_size = {k: v for k, v in size_obj.items() if k != 'purchase_links'} - non_refill_size['purchase_links'] = non_refill_links - # Ensure spool_refill is not set or is False - non_refill_size.pop('spool_refill', None) - new_sizes.append(non_refill_size) - - if refill_links: - # Refill size (with spool_refill=true at size level) - refill_size = {k: v for k, v in size_obj.items() if k != 'purchase_links'} - refill_size['spool_refill'] = True - refill_size['purchase_links'] = refill_links - new_sizes.append(refill_size) - changed = True - - return new_sizes, changed - - -def migrate_sizes( - sizes_file: Path, - id_mapping: dict[str, str], - valid_store_ids: set[str], - dry_run: bool -) -> bool: - """Update store_id references and migrate spool_refill in a sizes.json file. - - Args: - sizes_file: Path to the sizes.json file. - id_mapping: Dictionary mapping old store IDs to new store IDs. - valid_store_ids: Set of valid store IDs from stores directory. - dry_run: If True, don't actually modify files. - - Returns: - True if changes were made, False otherwise. - """ - data = load_json(sizes_file) - if data is None: - return False - - file_changed = False - - # sizes.json is a list of size objects - if isinstance(data, list): - # First, migrate spool_refill from purchase_link level to size level - data, spool_refill_changed = migrate_spool_refill(data) - if spool_refill_changed: - print(f" Migrated spool_refill to size level") - file_changed = True - - # Then update store_id references - for size_obj in data: - if not isinstance(size_obj, dict): - continue - purchase_links = size_obj.get('purchase_links', []) - if not isinstance(purchase_links, list): - continue - - for link in purchase_links: - if not isinstance(link, dict): - continue - old_store_id = link.get('store_id') - if not old_store_id: - continue - - # Find the matching store ID - new_store_id = find_matching_store_id( - old_store_id, id_mapping, valid_store_ids - ) - - if new_store_id is None: - # No match found - will be reported by caller - continue - - if old_store_id != new_store_id: - link['store_id'] = new_store_id - print(f" Updated store_id: {old_store_id} -> {new_store_id}") - file_changed = True - - if file_changed: - save_json(sizes_file, data, dry_run) - - return file_changed - - -def merge_variant_dirs( - source_dir: Path, target_dir: Path, dry_run: bool -) -> bool: - """Merge data from source variant directory into target if target is missing data. - - Args: - source_dir: The source variant directory (will be deleted after merge). - target_dir: The target variant directory to merge into. - dry_run: If True, don't actually modify files. - - Returns: - True if merge was successful (source can be deleted), False otherwise. - """ - # Merge variant.json data if target is missing fields - source_variant = load_json(source_dir / 'variant.json') - target_variant = load_json(target_dir / 'variant.json') - - if source_variant and target_variant: - merged = False - for key, value in source_variant.items(): - if key not in target_variant and key != 'id': - target_variant[key] = value - merged = True - if dry_run: - print(f" Would merge field '{key}' into target variant.json") - else: - print(f" Merged field '{key}' into target variant.json") - - if merged and not dry_run: - save_json(target_dir / 'variant.json', target_variant, dry_run) - - # Merge sizes.json if source has data target doesn't - source_sizes = source_dir / 'sizes.json' - target_sizes = target_dir / 'sizes.json' - - if source_sizes.exists(): - source_data = load_json(source_sizes) - target_data = load_json(target_sizes) if target_sizes.exists() else [] - - if source_data and isinstance(source_data, list): - if not target_data: - # Target has no sizes, copy from source - if dry_run: - print(f" Would copy sizes.json to target") - else: - shutil.copy2(str(source_sizes), str(target_sizes)) - print(f" Copied sizes.json to target") - else: - if dry_run: - print(f" Would skip sizes.json (target already has data)") - else: - print(f" Skipped sizes.json (target already has data)") - - return True - - -def migrate_variant( - variant_dir: Path, - dry_run: bool, - id_mapping: dict[str, str] | None = None, - valid_store_ids: set[str] | None = None -) -> tuple[bool, Path]: - """Transform variant.json: color_name->name, add id, rename folder. - Also updates store_id references in sizes.json. - - If a folder with the target name already exists (e.g., renaming "Black-White" to - "black_white" when "black_white" already exists), merge the contents and delete the source. - - Args: - variant_dir: Path to the variant directory. - dry_run: If True, don't actually modify files. - id_mapping: Dictionary mapping old store IDs to new store IDs. - valid_store_ids: Set of valid store IDs from stores directory. - - Returns: - Tuple of (changes_made, new_path after folder rename). - """ - variant_file = variant_dir / 'variant.json' - data = load_json(variant_file) - if data is None: - return False, variant_dir - - changed = False - folder_name = variant_dir.name - new_id = generate_id(folder_name) - - # Add id if missing - if 'id' not in data: - data['id'] = new_id - print(f" Added id: {data['id']}") - changed = True - - # Rename 'color_name' to 'name' - if 'color_name' in data: - data['name'] = data.pop('color_name') - print(f" Renamed 'color_name' -> 'name': {data['name']}") - changed = True - - if changed: - # Reorder keys: id first, then name, then color_hex, then rest - ordered = {} - for key in ['id', 'name', 'color_hex']: - if key in data: - ordered[key] = data[key] - for key in data: - if key not in ordered: - ordered[key] = data[key] - save_json(variant_file, ordered, dry_run) - - # Update store_id references in sizes.json - sizes_file = variant_dir / 'sizes.json' - if sizes_file.exists() and id_mapping is not None and valid_store_ids is not None: - if migrate_sizes(sizes_file, id_mapping, valid_store_ids, dry_run): - changed = True - - # Check if target folder already exists (duplicate scenario) - target_dir = variant_dir.parent / new_id - if target_dir.exists() and target_dir != variant_dir: - # Target exists - merge into it and delete source - print(f" Merging duplicate variant: {folder_name} -> {new_id}") - merge_variant_dirs(variant_dir, target_dir, dry_run) - if dry_run: - print(f" Would delete merged source: {folder_name}") - else: - shutil.rmtree(str(variant_dir)) - print(f" Deleted merged source: {folder_name}") - return True, target_dir - - # Rename folder to match id - new_dir = rename_folder(variant_dir, new_id, dry_run) - if new_dir != variant_dir: - changed = True - - return changed, new_dir - - -def merge_filament_dirs( - source_dir: Path, target_dir: Path, dry_run: bool -) -> bool: - """Merge data from source filament directory into target if target is missing data. - - Compares the two directories and merges any missing variants or data from source - into target. After merging, the source can be safely deleted. - - Args: - source_dir: The source filament directory (will be deleted after merge). - target_dir: The target filament directory to merge into. - dry_run: If True, don't actually modify files. - - Returns: - True if merge was successful (source can be deleted), False otherwise. - """ - # Get variants from both directories - source_variants = {d.name: d for d in source_dir.iterdir() if d.is_dir()} - target_variants = {d.name: d for d in target_dir.iterdir() if d.is_dir()} - - # Merge missing variants from source to target - for variant_name, variant_dir in source_variants.items(): - if variant_name not in target_variants: - # Variant doesn't exist in target - move it - dest = target_dir / variant_name - if dry_run: - print(f" Would move variant: {variant_name} -> {target_dir.name}/") - else: - shutil.move(str(variant_dir), str(dest)) - print(f" Moved variant: {variant_name} -> {target_dir.name}/") - else: - # Variant exists in both - check if we need to merge sizes.json - source_sizes = variant_dir / 'sizes.json' - target_sizes = target_variants[variant_name] / 'sizes.json' - - if source_sizes.exists(): - source_data = load_json(source_sizes) - target_data = load_json(target_sizes) if target_sizes.exists() else [] - - if source_data and target_data is not None: - # Check for purchase links or sizes that might be missing - # For simplicity, if source has data target doesn't, log it - if dry_run: - print(f" Would skip duplicate variant: {variant_name} (exists in target)") - else: - print(f" Skipped duplicate variant: {variant_name} (exists in target)") - - # Merge filament.json data if target is missing fields - source_filament = load_json(source_dir / 'filament.json') - target_filament = load_json(target_dir / 'filament.json') - - if source_filament and target_filament: - merged = False - for key, value in source_filament.items(): - if key not in target_filament and key != 'id': - target_filament[key] = value - merged = True - if dry_run: - print(f" Would merge field '{key}' into target filament.json") - else: - print(f" Merged field '{key}' into target filament.json") - - if merged and not dry_run: - save_json(target_dir / 'filament.json', target_filament, dry_run) - - return True - - -def migrate_filament(filament_dir: Path, dry_run: bool) -> tuple[bool, Path]: - """Transform filament.json: add id, rename folder. - - If a folder with the target name already exists (e.g., renaming "Rigid X" to - "rigid_x" when "rigid_x" already exists), merge the contents and delete the source. - - Returns: - Tuple of (changes_made, new_path after folder rename). - """ - filament_file = filament_dir / 'filament.json' - data = load_json(filament_file) - if data is None: - return False, filament_dir - - changed = False - folder_name = filament_dir.name - new_id = generate_id(folder_name) - - # Add or update id to match folder - if data.get('id') != new_id: - old_id = data.get('id') - data['id'] = new_id - if old_id: - print(f" Updated id: {old_id} -> {new_id}") - else: - print(f" Added id: {new_id}") - changed = True - - if changed: - # Reorder keys: id first, then name, then rest - ordered = {} - for key in ['id', 'name']: - if key in data: - ordered[key] = data[key] - for key in data: - if key not in ordered: - ordered[key] = data[key] - save_json(filament_file, ordered, dry_run) - - # Check if target folder already exists (duplicate scenario) - target_dir = filament_dir.parent / new_id - if target_dir.exists() and target_dir != filament_dir: - # Target exists - merge into it and delete source - print(f" Merging duplicate: {folder_name} -> {new_id}") - merge_filament_dirs(filament_dir, target_dir, dry_run) - if dry_run: - print(f" Would delete merged source: {folder_name}") - else: - shutil.rmtree(str(filament_dir)) - print(f" Deleted merged source: {folder_name}") - return True, target_dir - - # Rename folder to match id - new_dir = rename_folder(filament_dir, new_id, dry_run) - if new_dir != filament_dir: - changed = True - - return changed, new_dir - - -def migrate_material( - material_dir: Path, dry_run: bool, report: MigrationReport -) -> tuple[bool, Path | None]: - """Transform material.json: map material types to enum values, rename folder. - - Note: material.json only has 'material', 'default_max_dry_temperature', - and 'default_slicer_settings' fields - no 'id' field. - - If the target material folder already exists (e.g., mapping APLA -> PLA when - PLA folder exists), moves all filament subdirectories to the target folder - and deletes the source folder. - - Returns: - Tuple of (changes_made, new_path after folder rename). - If the folder was merged into another and deleted, returns (True, None). - """ - material_file = material_dir / 'material.json' - data = load_json(material_file) - if data is None: - return False, material_dir - - changed = False - folder_name = material_dir.name - original_material = data.get('material', folder_name) - - # Remove 'id' field if present (not part of material schema) - if 'id' in data: - del data['id'] - print(f" Removed invalid 'id' field from material.json") - changed = True - - # Map material type to enum value if needed - new_material = original_material - if original_material in MATERIAL_TYPE_MAP: - new_material = MATERIAL_TYPE_MAP[original_material] - data['material'] = new_material - print(f" Mapped material: {original_material} -> {new_material}") - changed = True - else: - # Check if material is already a valid enum value (all caps typically) - # If not, record it as an unmapped material - material_value = data.get('material', '') - if material_value and not material_value.isupper(): - # Likely needs mapping but we don't have one - report.add_issue( - "Unmapped material type", - str(material_dir.relative_to(material_dir.parent.parent)), - f"Material '{material_value}' is not mapped to a schema enum value" - ) - - # Check if target folder already exists (merge scenario) - new_folder_name = data.get('material', folder_name) - target_dir = material_dir.parent / new_folder_name - - if target_dir.exists() and target_dir != material_dir: - # Target folder exists - merge filaments into it - print(f" Merging into existing folder: {new_folder_name}") - - # Build a mapping of normalized names to actual folder names in target - target_filaments: dict[str, Path] = {} - for d in target_dir.iterdir(): - if d.is_dir(): - # Map both the actual name and the normalized name - target_filaments[d.name] = d - target_filaments[generate_id(d.name)] = d - - # Move all filament subdirectories to target - filament_dirs = [d for d in material_dir.iterdir() if d.is_dir()] - for filament_dir in filament_dirs: - # Check if a matching filament exists in target (by name or normalized name) - source_name = filament_dir.name - source_normalized = generate_id(source_name) - - matching_target = target_filaments.get(source_name) or target_filaments.get(source_normalized) - - if matching_target: - # Filament exists in both - merge data from source to target - print(f" Merging duplicate filament: {source_name} -> {matching_target.name}") - merge_filament_dirs(filament_dir, matching_target, dry_run) - # Delete the source filament directory after merge - if dry_run: - print(f" Would delete merged source: {source_name}") - else: - shutil.rmtree(str(filament_dir)) - print(f" Deleted merged source: {source_name}") - else: - # Filament doesn't exist in target - move it - dest = target_dir / source_name - if dry_run: - print(f" Would move filament: {source_name} -> {new_folder_name}/") - else: - shutil.move(str(filament_dir), str(dest)) - print(f" Moved filament: {source_name} -> {new_folder_name}/") - - # Delete the now-empty source folder (including material.json) - if dry_run: - print(f" Would delete empty folder: {folder_name}") - else: - shutil.rmtree(str(material_dir)) - print(f" Deleted empty folder: {folder_name}") - - return True, None # Signal that folder was merged and deleted - - # Normal case: just update material.json and rename folder - if changed: - # Reorder keys: material first, then rest - ordered = {} - if 'material' in data: - ordered['material'] = data['material'] - for key in data: - if key not in ordered: - ordered[key] = data[key] - save_json(material_file, ordered, dry_run) - - # Rename folder to match material type (keep uppercase, e.g., "PLA", "CF") - new_dir = rename_folder(material_dir, new_folder_name, dry_run) - if new_dir != material_dir: - changed = True - - return changed, new_dir - - -def migrate_brand(brand_dir: Path, dry_run: bool) -> tuple[bool, Path]: - """Transform brand.json: brand->name, add id, rename logo, rename folder. - - Returns: - Tuple of (changes_made, new_path after folder rename). - """ - brand_file = brand_dir / 'brand.json' - data = load_json(brand_file) - if data is None: - return False, brand_dir - - changed = False - folder_name = brand_dir.name - new_id = generate_id(folder_name) - - # Add id if missing - if 'id' not in data: - data['id'] = new_id - print(f" Added id: {data['id']}") - changed = True - - # Rename 'brand' to 'name' - if 'brand' in data: - data['name'] = data.pop('brand') - print(f" Renamed 'brand' -> 'name': {data['name']}") - changed = True - - # Rename logo file and update reference - if 'logo' in data: - new_logo = rename_logo(brand_dir, data['logo'], dry_run) - if new_logo and new_logo != data['logo']: - data['logo'] = new_logo - changed = True - - if changed: - # Reorder keys: id first, then name, then rest - ordered = {} - for key in ['id', 'name', 'website', 'logo', 'origin']: - if key in data: - ordered[key] = data[key] - for key in data: - if key not in ordered: - ordered[key] = data[key] - save_json(brand_file, ordered, dry_run) - - # Rename folder to match id - new_dir = rename_folder(brand_dir, new_id, dry_run) - if new_dir != brand_dir: - changed = True - - return changed, new_dir - - -def migrate_store( - store_dir: Path, dry_run: bool, id_mapping: dict[str, str] -) -> tuple[bool, Path]: - """Transform store.json: rename logo file, rename folder. - - Args: - store_dir: Path to the store directory. - dry_run: If True, don't actually modify files. - id_mapping: Dictionary to populate with old_id -> new_id mappings. - - Returns: - Tuple of (changes_made, new_path after folder rename). - """ - store_file = store_dir / 'store.json' - data = load_json(store_file) - if data is None: - return False, store_dir - - changed = False - folder_name = store_dir.name - new_id = generate_id(folder_name) - - # Record the old ID -> new ID mapping - old_id = data.get('id', folder_name) - if old_id != new_id: - id_mapping[old_id] = new_id - # Also map the folder name if different from the old id - if folder_name != old_id and folder_name != new_id: - id_mapping[folder_name] = new_id - - # Update id in data if it doesn't match the new format - if 'id' in data and data['id'] != new_id: - data['id'] = new_id - print(f" Updated id: {data['id']}") - changed = True - - # Rename logo file and update reference - if 'logo' in data: - new_logo = rename_logo(store_dir, data['logo'], dry_run) - if new_logo and new_logo != data['logo']: - data['logo'] = new_logo - changed = True - - if changed: - save_json(store_file, data, dry_run) - - # Rename folder to match id - new_dir = rename_folder(store_dir, new_id, dry_run) - if new_dir != store_dir: - changed = True - - return changed, new_dir - - -def process_data_directory( - data_dir: Path, - dry_run: bool, - report: MigrationReport, - id_mapping: dict[str, str] | None = None, - valid_store_ids: set[str] | None = None -) -> dict[str, int]: - """Process all data files in the data directory. - - Important: We process from deepest level up (variants -> filaments -> materials -> brands) - to avoid path issues when renaming folders. - - Args: - data_dir: Path to the data directory. - dry_run: If True, don't actually modify files. - report: MigrationReport to add issues to. - id_mapping: Dictionary mapping old store IDs to new store IDs. - valid_store_ids: Set of valid store IDs from stores directory. - - Returns: - Dictionary with counts of modified files by type. - """ - stats = {'brands': 0, 'materials': 0, 'filaments': 0, 'variants': 0} - - # Collect all directories first to avoid issues with iteration during rename - brand_dirs = sorted([d for d in data_dir.iterdir() if d.is_dir()]) - - for brand_dir in brand_dirs: - print(f"\nBrand: {brand_dir.name}") - - # First, process all nested content (variants -> filaments -> materials) - material_dirs = sorted([d for d in brand_dir.iterdir() if d.is_dir()]) - - for material_dir in material_dirs: - # Check if this is a material directory (has material.json) - if not (material_dir / 'material.json').exists(): - continue - - print(f" Material: {material_dir.name}") - - filament_dirs = sorted([d for d in material_dir.iterdir() if d.is_dir()]) - - for filament_dir in filament_dirs: - # Check if this is a filament directory (has filament.json) - if not (filament_dir / 'filament.json').exists(): - continue - - print(f" Filament: {filament_dir.name}") - - # Process variants FIRST (deepest level) - variant_dirs = sorted([d for d in filament_dir.iterdir() if d.is_dir()]) - for variant_dir in variant_dirs: - if (variant_dir / 'variant.json').exists(): - print(f" Variant: {variant_dir.name}") - changed, _ = migrate_variant( - variant_dir, dry_run, id_mapping, valid_store_ids - ) - if changed: - stats['variants'] += 1 - - # Then process filament - changed, _ = migrate_filament(filament_dir, dry_run) - if changed: - stats['filaments'] += 1 - - # Then process material - changed, _ = migrate_material(material_dir, dry_run, report) - if changed: - stats['materials'] += 1 - - # Finally, process the brand itself - changed, _ = migrate_brand(brand_dir, dry_run) - if changed: - stats['brands'] += 1 - - return stats - - -def process_stores_directory( - stores_dir: Path, dry_run: bool -) -> tuple[int, dict[str, str]]: - """Process all store files in the stores directory. - - Returns: - Tuple of (count of modified store files, old_id -> new_id mapping). - """ - count = 0 - id_mapping: dict[str, str] = {} - - # Collect directories first to avoid iteration issues - store_dirs = sorted([d for d in stores_dir.iterdir() if d.is_dir()]) - - for store_dir in store_dirs: - print(f"\nStore: {store_dir.name}") - changed, _ = migrate_store(store_dir, dry_run, id_mapping) - if changed: - count += 1 - - return count, id_mapping - - -def get_valid_store_ids(stores_dir: Path) -> set[str]: - """Scan stores directory and return all valid store IDs. - - Returns: - Set of valid store IDs (folder names that contain store.json). - """ - valid_ids = set() - if not stores_dir.exists(): - return valid_ids - - for store_dir in stores_dir.iterdir(): - if store_dir.is_dir() and (store_dir / 'store.json').exists(): - valid_ids.add(store_dir.name) - - return valid_ids - - -def find_matching_store_id( - old_store_id: str, - id_mapping: dict[str, str], - valid_store_ids: set[str] -) -> str | None: - """Find the correct store ID for an old store_id reference. - - Tries multiple strategies to find a match: - 1. Check explicit id_mapping from migration - 2. Check if it's already a valid store ID - 3. Check if generate_id(old_store_id) matches a valid store - 4. Check if any valid store ID contains the normalized old_store_id - - Args: - old_store_id: The store_id value found in data files. - id_mapping: Dictionary mapping old store IDs to new store IDs. - valid_store_ids: Set of valid store IDs from stores directory. - - Returns: - The matching store ID, or None if no match found. - """ - # 1. Check explicit mapping from migration - if old_store_id in id_mapping: - return id_mapping[old_store_id] - - # 2. Already a valid store ID - if old_store_id in valid_store_ids: - return old_store_id - - # 3. Check if normalized form is valid - normalized = generate_id(old_store_id) - if normalized in valid_store_ids: - return normalized - - # 4. Try fuzzy matching - check if removing common suffixes helps - # e.g., "3do.dk" -> check if "3do" exists - for suffix in ['.dk', '.com', '.de', '.eu', '_amazon', '_store']: - if old_store_id.lower().endswith(suffix): - base = old_store_id[:-len(suffix)] - base_normalized = generate_id(base) - if base_normalized in valid_store_ids: - return base_normalized - - # 5. Check case-insensitive match - old_lower = old_store_id.lower() - for valid_id in valid_store_ids: - if valid_id.lower() == old_lower: - return valid_id - - return None - - -def update_store_references( - data_dir: Path, - stores_dir: Path, - id_mapping: dict[str, str], - dry_run: bool, - report: MigrationReport -) -> int: - """Update store_id references and migrate spool_refill in all sizes.json files. - - Normalizes store_id values to match existing store IDs. - Uses multiple strategies to find the correct store ID: - 1. Explicit mappings discovered during store migration - 2. Direct match with existing store IDs - 3. Normalized form matching - 4. Fuzzy matching for common patterns - - Also migrates spool_refill from purchase_link level to size level. - - Args: - data_dir: Path to the data directory. - stores_dir: Path to the stores directory. - id_mapping: Dictionary mapping old store IDs to new store IDs - (built dynamically during store migration). - dry_run: If True, don't actually modify files. - report: MigrationReport to add issues to. - - Returns: - Count of sizes.json files updated. - """ - print(f"\nUpdating store_id references...") - - # Get all valid store IDs from the stores directory - valid_store_ids = get_valid_store_ids(stores_dir) - print(f" Found {len(valid_store_ids)} valid stores") - - if id_mapping: - print(f" Discovered mappings: {id_mapping}") - - count = 0 - unmatched_stores: set[str] = set() - - # Find all sizes.json files - for sizes_file in data_dir.rglob('sizes.json'): - data = load_json(sizes_file) - if data is None: - continue - - file_changed = False - - # sizes.json is a list of size objects - if isinstance(data, list): - # First, migrate spool_refill from purchase_link level to size level - data, spool_refill_changed = migrate_spool_refill(data) - if spool_refill_changed: - print(f" {sizes_file.relative_to(data_dir)}: migrated spool_refill") - file_changed = True - - # Then update store_id references - for size_obj in data: - if not isinstance(size_obj, dict): - continue - purchase_links = size_obj.get('purchase_links', []) - if not isinstance(purchase_links, list): - continue - - for link in purchase_links: - if not isinstance(link, dict): - continue - old_store_id = link.get('store_id') - if not old_store_id: - continue - - # Find the matching store ID - new_store_id = find_matching_store_id( - old_store_id, id_mapping, valid_store_ids - ) - - if new_store_id is None: - # No match found - track for reporting - unmatched_stores.add(old_store_id) - continue - - if old_store_id != new_store_id: - link['store_id'] = new_store_id - print(f" {sizes_file.relative_to(data_dir)}: " - f"{old_store_id} -> {new_store_id}") - file_changed = True - - if file_changed: - save_json(sizes_file, data, dry_run) - count += 1 - - # Report unmatched stores - for store_id in sorted(unmatched_stores): - report.add_issue( - "Unknown store reference", - "sizes.json files", - f"store_id '{store_id}' does not match any known store" - ) - - return count - - -def main(): - parser = argparse.ArgumentParser( - description='Migrate data files to new schema format' - ) - parser.add_argument( - '--dry-run', - action='store_true', - help='Preview changes without modifying files' - ) - parser.add_argument( - '--data-only', - action='store_true', - help='Only process data directory (skip stores)' - ) - parser.add_argument( - '--stores-only', - action='store_true', - help='Only process stores directory (skip data)' - ) - args = parser.parse_args() - - # Determine paths relative to script location - script_dir = Path(__file__).parent - repo_root = script_dir.parent - data_dir = repo_root / 'data' - stores_dir = repo_root / 'stores' - - if args.dry_run: - print("=== DRY RUN MODE - No files will be modified ===\n") - - total_changes = 0 - report = MigrationReport() - - # Process stores directory FIRST to build the ID mapping - # This mapping is needed to update store_id references in data files - store_id_mapping: dict[str, str] = {} - valid_store_ids: set[str] = set() - if not args.data_only: - if stores_dir.exists(): - print("Processing stores directory...") - store_count, store_id_mapping = process_stores_directory( - stores_dir, args.dry_run - ) - # Get valid store IDs after processing (folder names after migration) - valid_store_ids = get_valid_store_ids(stores_dir) - total_changes += store_count - print(f"\n--- Stores Summary ---") - print(f" Stores modified: {store_count}") - print(f" Valid store IDs: {len(valid_store_ids)}") - if store_id_mapping: - print(f" ID mappings discovered: {len(store_id_mapping)}") - else: - print(f"Stores directory not found: {stores_dir}") - else: - # Even if skipping stores, we need valid IDs to update references - valid_store_ids = get_valid_store_ids(stores_dir) - - # Process data directory (includes store_id reference updates during variant migration) - if not args.stores_only: - if data_dir.exists(): - print("\nProcessing data directory...") - stats = process_data_directory( - data_dir, args.dry_run, report, store_id_mapping, valid_store_ids - ) - total_changes += sum(stats.values()) - print(f"\n--- Data Summary ---") - print(f" Brands modified: {stats['brands']}") - print(f" Materials modified: {stats['materials']}") - print(f" Filaments modified: {stats['filaments']}") - print(f" Variants modified: {stats['variants']}") - else: - print(f"Data directory not found: {data_dir}") - - print(f"\n{'=' * 40}") - if args.dry_run: - print(f"DRY RUN COMPLETE: {total_changes} items would be modified") - else: - print(f"MIGRATION COMPLETE: {total_changes} items modified") - - # Print any issues encountered - report.print_summary() - - -if __name__ == '__main__': - main() diff --git a/scripts/sort_data.py b/scripts/sort_data.py new file mode 100644 index 0000000000..fcce8b7024 --- /dev/null +++ b/scripts/sort_data.py @@ -0,0 +1,593 @@ +#!/usr/bin/env python3 +""" +sort_data.py - Sort JSON file keys according to schema definitions + +This script recursively processes all JSON files in the data/ and stores/ +directories and reorders their keys to match the order defined in the +corresponding JSON schemas. This ensures consistent formatting across all +data files. + +The script: +1. Loads all schemas and extracts property key orderings +2. Processes each JSON file and sorts keys according to schema +3. Handles nested objects with their own key orderings +4. Warns about keys found in data but not in schema +5. Validates all files after sorting using data_validator.py + +Usage: + python scripts/sort_data.py --dry-run # Preview changes + python scripts/sort_data.py # Apply changes +""" + +import argparse +import json +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Optional, Set + +# Add parent directory to path to import data_validator +sys.path.insert(0, str(Path(__file__).parent.parent)) +from data_validator import ValidationOrchestrator + +# Global flags for output mode +PROGRESS_MODE = False +JSON_MODE = False + + +def emit_progress(stage: str, percent: int, message: str = '') -> None: + """Emit progress event as JSON to stdout for SSE streaming.""" + if PROGRESS_MODE and hasattr(sys.stdout, 'isatty') and not sys.stdout.isatty(): + # Only emit when stdout is piped (not terminal) + print(json.dumps({ + 'type': 'progress', + 'stage': stage, + 'percent': percent, + 'message': message + }), flush=True) + + +@dataclass +class SchemaInfo: + """Holds key ordering information for a schema.""" + keys: List[str] = field(default_factory=list) + nested: Dict[str, List[str]] = field(default_factory=dict) + + +@dataclass +class ProcessingStats: + """Statistics for file processing.""" + files_processed: int = 0 + files_modified: int = 0 + files_skipped: int = 0 + extra_keys_found: int = 0 + + def to_dict(self) -> Dict[str, int]: + """Convert to dictionary for JSON serialization.""" + return { + 'files_processed': self.files_processed, + 'files_modified': self.files_modified, + 'files_skipped': self.files_skipped, + 'extra_keys_found': self.extra_keys_found + } + + +def load_json(path: Path) -> Optional[Dict[str, Any]]: + """Load JSON from file with error handling.""" + try: + with open(path, 'r', encoding='utf-8') as f: + return json.load(f) + except (json.JSONDecodeError, OSError) as e: + print(f"Error loading {path}: {e}") + return None + + +def save_json(path: Path, data: Any, dry_run: bool) -> None: + """Save JSON to file with consistent formatting.""" + if dry_run: + return + with open(path, 'w', encoding='utf-8') as f: + json.dump(data, f, indent=2, ensure_ascii=False) + f.write('\n') + + +def load_schemas(schemas_dir: Path) -> Dict[str, Dict[str, Any]]: + """Load all JSON schemas from the schemas directory.""" + schemas = {} + + if not schemas_dir.exists(): + print(f"Error: {schemas_dir} directory not found") + return schemas + + for schema_file in schemas_dir.glob("*.json"): + try: + with open(schema_file, 'r', encoding='utf-8') as f: + schemas[schema_file.stem] = json.load(f) + except json.JSONDecodeError as e: + print(f"Error parsing {schema_file.name}: {e}") + + return schemas + + +def get_property_order(schema: Dict[str, Any]) -> List[str]: + """Extract the order of properties from a JSON schema.""" + if "properties" in schema: + return list(schema["properties"].keys()) + return [] + + +def extract_nested_schemas(schema: Dict[str, Any]) -> Dict[str, List[str]]: + """Recursively extract nested object schemas and their key orderings. + + Returns: + Dictionary mapping nested property names to their key orderings. + For example: {'color_standards': ['ral', 'ncs', 'pantone', ...]} + """ + nested = {} + + # Check properties in the main schema + if "properties" in schema: + for prop_name, prop_schema in schema["properties"].items(): + if isinstance(prop_schema, dict): + # Handle object types + if prop_schema.get("type") == "object" and "properties" in prop_schema: + nested[prop_name] = get_property_order(prop_schema) + # Handle array types with object items + elif prop_schema.get("type") == "array" and "items" in prop_schema: + items_schema = prop_schema["items"] + if isinstance(items_schema, dict) and items_schema.get("type") == "object": + if "properties" in items_schema: + nested[prop_name] = get_property_order(items_schema) + + # Check definitions section for nested schemas + if "definitions" in schema: + for def_name, def_schema in schema["definitions"].items(): + if isinstance(def_schema, dict) and def_schema.get("type") == "object": + if "properties" in def_schema: + nested[def_name] = get_property_order(def_schema) + + return nested + + +def build_key_order_map(schemas_dir: Path) -> Dict[str, SchemaInfo]: + """Build a mapping of schema names to their key orderings. + + Returns: + Dictionary mapping schema names (without _schema suffix) to SchemaInfo. + """ + schemas = load_schemas(schemas_dir) + key_order_map = {} + + for schema_name, schema_content in schemas.items(): + # Remove _schema suffix if present + clean_name = schema_name.replace('_schema', '') + + # Check if this is an array schema (like sizes_schema.json) + if schema_content.get('type') == 'array' and 'items' in schema_content: + # For array schemas, use the items schema + items_schema = schema_content['items'] + keys = get_property_order(items_schema) + nested = extract_nested_schemas(items_schema) + else: + # Extract top-level key order + keys = get_property_order(schema_content) + # Extract nested object key orderings + nested = extract_nested_schemas(schema_content) + + key_order_map[clean_name] = SchemaInfo(keys=keys, nested=nested) + + return key_order_map + + +def sort_json_keys( + data: Any, + schema_info: SchemaInfo, + extra_keys: Set[str] +) -> Any: + """Recursively sort JSON keys according to schema ordering. + + Args: + data: The JSON data to sort (dict, list, or primitive) + schema_info: Schema information with key orderings + extra_keys: Set to collect keys not in schema + + Returns: + Sorted JSON data + """ + if isinstance(data, dict): + ordered = {} + remaining_keys = set(data.keys()) + + # Add keys in schema order first + for key in schema_info.keys: + if key in data: + value = data[key] + + # Check if this key has a nested schema + if key in schema_info.nested: + nested_info = SchemaInfo(keys=schema_info.nested[key], nested=schema_info.nested) + + # If value is an array, process each item with nested schema + if isinstance(value, list): + value = [sort_json_keys(item, nested_info, extra_keys) if isinstance(item, dict) + else item for item in value] + else: + value = sort_json_keys(value, nested_info, extra_keys) + elif isinstance(value, dict): + # No specific nested schema, just recurse with empty schema + value = sort_json_keys(value, SchemaInfo(), extra_keys) + elif isinstance(value, list): + # Process list items + value = [sort_json_keys(item, schema_info, extra_keys) if isinstance(item, dict) + else item for item in value] + + ordered[key] = value + remaining_keys.remove(key) + + # Add remaining keys alphabetically (these are not in schema) + if remaining_keys: + extra_keys.update(remaining_keys) + for key in sorted(remaining_keys): + value = data[key] + if isinstance(value, dict): + value = sort_json_keys(value, SchemaInfo(), extra_keys) + elif isinstance(value, list): + value = [sort_json_keys(item, SchemaInfo(), extra_keys) if isinstance(item, dict) + else item for item in value] + ordered[key] = value + + return ordered + + elif isinstance(data, list): + # For arrays, process each item + # For sizes.json, each item should follow the size item schema + result = [] + for item in data: + if isinstance(item, dict): + # For array items, use the same schema_info + result.append(sort_json_keys(item, schema_info, extra_keys)) + elif isinstance(item, list): + result.append(sort_json_keys(item, schema_info, extra_keys)) + else: + result.append(item) + return result + + else: + return data + + +def process_json_file( + file_path: Path, + schema_name: str, + key_order_map: Dict[str, SchemaInfo], + dry_run: bool, + stats: ProcessingStats +) -> bool: + """Process a single JSON file and sort its keys. + + Args: + file_path: Path to the JSON file + schema_name: Name of the schema to use + key_order_map: Mapping of schema names to key orderings + dry_run: If True, don't save changes + stats: Statistics tracker + + Returns: + True if file was modified, False otherwise + """ + # Load the file + data = load_json(file_path) + if data is None: + stats.files_skipped += 1 + return False + + # Get schema info + if schema_name not in key_order_map: + if not JSON_MODE: + print(f" Warning: No schema found for {schema_name}") + stats.files_skipped += 1 + return False + + schema_info = key_order_map[schema_name] + + # Track extra keys found in this file + extra_keys: Set[str] = set() + + # Sort the keys + sorted_data = sort_json_keys(data, schema_info, extra_keys) + + # Warn about extra keys + if extra_keys: + if not JSON_MODE: + print(f" Warning: Extra keys in {file_path.name}: {sorted(extra_keys)}") + stats.extra_keys_found += len(extra_keys) + + # Check if anything changed + original_json = json.dumps(data, ensure_ascii=False, sort_keys=False) + sorted_json = json.dumps(sorted_data, ensure_ascii=False, sort_keys=False) + + stats.files_processed += 1 + + if original_json != sorted_json: + if not JSON_MODE: + if dry_run: + print(f" Would sort: {file_path.name}") + else: + print(f" Sorted: {file_path.name}") + save_json(file_path, sorted_data, dry_run) + stats.files_modified += 1 + return True + + return False + + +def process_data_directory( + data_dir: Path, + key_order_map: Dict[str, SchemaInfo], + dry_run: bool +) -> ProcessingStats: + """Process all JSON files in the data directory hierarchy. + + Args: + data_dir: Path to the data directory + key_order_map: Mapping of schema names to key orderings + dry_run: If True, don't save changes + + Returns: + Processing statistics + """ + stats = ProcessingStats() + + if not JSON_MODE: + print("Processing data directory...") + + # Process each brand directory + for brand_dir in sorted(data_dir.iterdir()): + if not brand_dir.is_dir(): + continue + + if not JSON_MODE: + print(f" Brand: {brand_dir.name}") + + # Process brand.json + brand_file = brand_dir / "brand.json" + if brand_file.exists(): + process_json_file(brand_file, "brand", key_order_map, dry_run, stats) + + # Process each material directory + for material_dir in sorted(brand_dir.iterdir()): + if not material_dir.is_dir(): + continue + + # Process material.json + material_file = material_dir / "material.json" + if material_file.exists(): + process_json_file(material_file, "material", key_order_map, dry_run, stats) + + # Process each filament directory + for filament_dir in sorted(material_dir.iterdir()): + if not filament_dir.is_dir(): + continue + + # Process filament.json + filament_file = filament_dir / "filament.json" + if filament_file.exists(): + process_json_file(filament_file, "filament", key_order_map, dry_run, stats) + + # Process each variant directory + for variant_dir in sorted(filament_dir.iterdir()): + if not variant_dir.is_dir(): + continue + + # Process variant.json + variant_file = variant_dir / "variant.json" + if variant_file.exists(): + process_json_file(variant_file, "variant", key_order_map, dry_run, stats) + + # Process sizes.json + sizes_file = variant_dir / "sizes.json" + if sizes_file.exists(): + process_json_file(sizes_file, "sizes", key_order_map, dry_run, stats) + + return stats + + +def process_stores_directory( + stores_dir: Path, + key_order_map: Dict[str, SchemaInfo], + dry_run: bool +) -> ProcessingStats: + """Process all JSON files in the stores directory. + + Args: + stores_dir: Path to the stores directory + key_order_map: Mapping of schema names to key orderings + dry_run: If True, don't save changes + + Returns: + Processing statistics + """ + stats = ProcessingStats() + + if not JSON_MODE: + print("\nProcessing stores directory...") + + # Process each store directory + for store_dir in sorted(stores_dir.iterdir()): + if not store_dir.is_dir(): + continue + + if not JSON_MODE: + print(f" Store: {store_dir.name}") + + # Process store.json + store_file = store_dir / "store.json" + if store_file.exists(): + process_json_file(store_file, "store", key_order_map, dry_run, stats) + + return stats + + +def merge_stats(stats1: ProcessingStats, stats2: ProcessingStats) -> ProcessingStats: + """Merge two ProcessingStats objects.""" + return ProcessingStats( + files_processed=stats1.files_processed + stats2.files_processed, + files_modified=stats1.files_modified + stats2.files_modified, + files_skipped=stats1.files_skipped + stats2.files_skipped, + extra_keys_found=stats1.extra_keys_found + stats2.extra_keys_found + ) + + +def main(): + global PROGRESS_MODE, JSON_MODE + + parser = argparse.ArgumentParser( + description='Sort JSON file keys according to schema definitions' + ) + parser.add_argument( + '--dry-run', + action='store_true', + help='Preview changes without modifying files' + ) + parser.add_argument( + '--json', + action='store_true', + help='Output results as JSON' + ) + parser.add_argument( + '--progress', + action='store_true', + help='Emit progress events (for SSE streaming)' + ) + parser.add_argument( + '--validate', + action='store_true', + help='Run validation after sorting' + ) + args = parser.parse_args() + + PROGRESS_MODE = args.progress + JSON_MODE = args.json + + # Determine paths relative to script location + script_dir = Path(__file__).parent + repo_root = script_dir.parent + data_dir = repo_root / 'data' + stores_dir = repo_root / 'stores' + schemas_dir = repo_root / 'schemas' + + if args.dry_run: + if not JSON_MODE: + print("=== DRY RUN MODE - No files will be modified ===\n") + + # Build key order mapping from schemas + emit_progress('loading_schemas', 0, 'Loading schemas...') + if not JSON_MODE: + print("Loading schemas...") + key_order_map = build_key_order_map(schemas_dir) + emit_progress('loading_schemas', 100, f'Loaded {len(key_order_map)} schemas') + if not JSON_MODE: + print(f"Loaded {len(key_order_map)} schemas\n") + + # Process data directory + data_stats = ProcessingStats() + if data_dir.exists(): + emit_progress('sorting_data', 0, 'Processing data directory...') + data_stats = process_data_directory(data_dir, key_order_map, args.dry_run) + emit_progress('sorting_data', 100, 'Data directory processing complete') + else: + if not JSON_MODE: + print(f"Data directory not found: {data_dir}") + + # Process stores directory + stores_stats = ProcessingStats() + if stores_dir.exists(): + emit_progress('sorting_stores', 0, 'Processing stores directory...') + stores_stats = process_stores_directory(stores_dir, key_order_map, args.dry_run) + emit_progress('sorting_stores', 100, 'Stores directory processing complete') + else: + if not JSON_MODE: + print(f"Stores directory not found: {stores_dir}") + + # Merge statistics + total_stats = merge_stats(data_stats, stores_stats) + + # Run validation if requested + validation_result = None + if args.validate and not args.dry_run and total_stats.files_modified > 0: + emit_progress('validation', 0, 'Running validation...') + if not JSON_MODE: + print(f"\n{'=' * 60}") + print("VALIDATING SORTED FILES") + print('=' * 60) + + orchestrator = ValidationOrchestrator(data_dir, stores_dir, progress_mode=PROGRESS_MODE) + validation_result = orchestrator.validate_all() + emit_progress('validation', 100, 'Validation complete') + + # Output results + if JSON_MODE: + # JSON output mode + output = { + 'dry_run': args.dry_run, + 'stats': total_stats.to_dict() + } + if validation_result: + output['validation'] = validation_result.to_dict() + + # Use compact output in progress mode for SSE compatibility, pretty output otherwise + if PROGRESS_MODE: + print(json.dumps(output)) + else: + print(json.dumps(output, indent=2)) + + # Exit with error code if validation failed + if validation_result and not validation_result.is_valid: + sys.exit(1) + sys.exit(0) + else: + # Text output mode + print(f"\n{'=' * 60}") + if args.dry_run: + print("DRY RUN SUMMARY") + else: + print("SORTING SUMMARY") + print('=' * 60) + print(f"Files processed: {total_stats.files_processed}") + print(f"Files modified: {total_stats.files_modified}") + print(f"Files skipped: {total_stats.files_skipped}") + if total_stats.extra_keys_found > 0: + print(f"Extra keys found: {total_stats.extra_keys_found}") + + if validation_result: + if validation_result.errors: + print("\nValidation errors found:") + + # Group errors by category + errors_by_category: Dict[str, List] = {} + for error in validation_result.errors: + if error.category not in errors_by_category: + errors_by_category[error.category] = [] + errors_by_category[error.category].append(error) + + # Print errors grouped by category + for category, errors in sorted(errors_by_category.items()): + print(f"\n{category} ({len(errors)}):") + print("-" * 80) + for error in errors[:10]: # Limit to first 10 per category + print(f" {error}") + if len(errors) > 10: + print(f" ... and {len(errors) - 10} more") + + print(f"\nValidation failed: {validation_result.error_count} errors, {validation_result.warning_count} warnings") + print("\nDone!") + sys.exit(1) + else: + print("\nAll validations passed!") + + print("\nDone!") + sys.exit(0) + + +if __name__ == '__main__': + main() diff --git a/webui/src/lib/components/SortDataButton.svelte b/webui/src/lib/components/SortDataButton.svelte new file mode 100644 index 0000000000..dc68a9c986 --- /dev/null +++ b/webui/src/lib/components/SortDataButton.svelte @@ -0,0 +1,72 @@ + + + + + diff --git a/webui/src/lib/components/ValidationDropdown.svelte b/webui/src/lib/components/ValidationDropdown.svelte new file mode 100644 index 0000000000..a13426b608 --- /dev/null +++ b/webui/src/lib/components/ValidationDropdown.svelte @@ -0,0 +1,285 @@ + + + + +
+ + + {#if isOpen} + + {/if} +
+ + diff --git a/webui/src/lib/components/ValidationProgressModal.svelte b/webui/src/lib/components/ValidationProgressModal.svelte new file mode 100644 index 0000000000..67d10464cf --- /dev/null +++ b/webui/src/lib/components/ValidationProgressModal.svelte @@ -0,0 +1,109 @@ + + +{#if isOpen} +
+
e.stopPropagation()} + role="dialog" + aria-labelledby="validation-progress-title" + aria-modal="true" + tabindex="-1" + > +

+ {jobType === 'validation' ? 'Running Validation...' : 'Sorting Data...'} +

+ + {#if error} +
+ Error: + {error} +
+ {:else} +
+
+
+
+

+ {progress.stage || 'Initializing...'} +

+ {#if progress.message} +

{progress.message}

+ {/if} +
+ +
+
+
+
+ {/if} + + +
+
+{/if} diff --git a/webui/src/lib/components/ValidationTriggerButton.svelte b/webui/src/lib/components/ValidationTriggerButton.svelte new file mode 100644 index 0000000000..499c1de355 --- /dev/null +++ b/webui/src/lib/components/ValidationTriggerButton.svelte @@ -0,0 +1,138 @@ + + + + + diff --git a/webui/src/lib/globalHelpers.ts b/webui/src/lib/globalHelpers.ts index 0a7db20c81..71a0577ef6 100644 --- a/webui/src/lib/globalHelpers.ts +++ b/webui/src/lib/globalHelpers.ts @@ -93,7 +93,8 @@ export const getIdFromName = (name: string): string => { name .trim() .toLowerCase() - .replace(/\s+/g, '_'), + .replace(/\s+/g, '_') + .replace(/-/g, '_'), ['+'] ); }; diff --git a/webui/src/lib/hooks/useSSE.ts b/webui/src/lib/hooks/useSSE.ts new file mode 100644 index 0000000000..bf8b2f1cff --- /dev/null +++ b/webui/src/lib/hooks/useSSE.ts @@ -0,0 +1,49 @@ +export interface SSEHandlers { + onProgress?: (data: any) => void; + onComplete?: (data: any) => void; + onError?: (error: any) => void; +} + +export function useSSE() { + let eventSource: EventSource | null = null; + + function connect(url: string, handlers: SSEHandlers) { + // Close any existing connection + disconnect(); + + eventSource = new EventSource(url); + + eventSource.onmessage = (event) => { + try { + const data = JSON.parse(event.data); + + if (data.type === 'progress' && handlers.onProgress) { + handlers.onProgress(data); + } else if (data.type === 'complete' && handlers.onComplete) { + handlers.onComplete(data.result); + disconnect(); + } else if (data.type === 'error' && handlers.onError) { + handlers.onError(data); + disconnect(); + } + } catch (e) { + console.error('SSE parse error:', e); + } + }; + + eventSource.onerror = (error) => { + console.error('SSE connection error:', error); + handlers.onError?.(error); + disconnect(); + }; + } + + function disconnect() { + if (eventSource) { + eventSource.close(); + eventSource = null; + } + } + + return { connect, disconnect }; +} diff --git a/webui/src/lib/server/jobManager.ts b/webui/src/lib/server/jobManager.ts new file mode 100644 index 0000000000..deae2b9425 --- /dev/null +++ b/webui/src/lib/server/jobManager.ts @@ -0,0 +1,109 @@ +// Shared job storage for validation and sort operations + +export interface Job { + id: string; + type: 'validation' | 'sort'; + startTime: number; + status: 'running' | 'complete' | 'error'; + events: any[]; + process?: any; + result?: any; + endTime?: number; +} + +// In-memory job storage +export const activeJobs = new Map(); + +// Lock for atomic validation job acquisition +let validationLock = false; + +// Helper functions for validation job management +export function getActiveValidationJob(): Job | null { + for (const job of activeJobs.values()) { + if (job.type === 'validation' && job.status === 'running') { + return job; + } + } + return null; +} + +export function hasActiveValidationJob(): boolean { + return getActiveValidationJob() !== null; +} + +/** + * Atomically try to acquire the validation lock. + * Returns true if lock was acquired, false if already locked. + */ +export function tryAcquireValidationLock(): boolean { + if (validationLock) { + return false; + } + validationLock = true; + return true; +} + +/** + * Release the validation lock. + */ +export function releaseValidationLock(): void { + validationLock = false; +} + +// Cleanup old jobs (older than 5 minutes) and timeout stuck jobs +// Store interval ID to allow cleanup on module reload +let cleanupIntervalId: NodeJS.Timeout | null = null; + +/** + * Stop the cleanup interval (useful for testing or hot reload scenarios) + */ +export function stopCleanupInterval(): void { + if (cleanupIntervalId) { + clearInterval(cleanupIntervalId); + cleanupIntervalId = null; + } +} + +/** + * Start the cleanup interval if not already running + */ +function startCleanupInterval(): void { + // Prevent multiple intervals in hot reload scenarios + if (cleanupIntervalId) { + return; + } + + cleanupIntervalId = setInterval(() => { + const now = Date.now(); + for (const [jobId, job] of activeJobs.entries()) { + // Remove old completed/errored jobs + if (job.endTime && now - job.endTime > 5 * 60 * 1000) { + activeJobs.delete(jobId); + } + + // Timeout jobs running for more than 30 minutes + if (job.status === 'running' && now - job.startTime > 30 * 60 * 1000) { + console.error(`Job ${jobId} timed out after 30 minutes`); + job.status = 'error'; + job.events.push({ + type: 'error', + message: 'Job timed out after 30 minutes' + }); + job.endTime = now; + + // Kill the process if it exists + if (job.process?.kill) { + job.process.kill('SIGTERM'); + } + + // Release validation lock if this was a validation job + if (job.type === 'validation') { + releaseValidationLock(); + } + } + } + }, 60 * 1000); // Run every minute +} + +// Start cleanup interval on module load +startCleanupInterval(); diff --git a/webui/src/lib/server/validationTrigger.ts b/webui/src/lib/server/validationTrigger.ts new file mode 100644 index 0000000000..9cc17c0fce --- /dev/null +++ b/webui/src/lib/server/validationTrigger.ts @@ -0,0 +1,118 @@ +import { spawn } from 'node:child_process'; +import path from 'node:path'; +import { activeJobs, type Job, tryAcquireValidationLock, releaseValidationLock } from './jobManager'; + +export async function triggerBackgroundValidation(): Promise { + // Atomically try to acquire the validation lock to prevent race conditions + if (!tryAcquireValidationLock()) { + console.log('[ValidationTrigger] Validation already running, skipping'); + return false; + } + + const jobId = 'validation-current'; + + // Clean up old validation-current job if it exists and is complete + const existingJob = activeJobs.get(jobId); + if (existingJob && (existingJob.status === 'complete' || existingJob.status === 'error')) { + activeJobs.delete(jobId); + } + + // Build Python command arguments + const args = ['data_validator.py', '--json', '--progress']; + const repoRoot = path.resolve(process.cwd(), '..'); + + // Store job info + const job: Job = { + id: jobId, + type: 'validation', + startTime: Date.now(), + status: 'running', + events: [] + }; + activeJobs.set(jobId, job); + console.log('[ValidationTrigger] Starting background validation'); + + // Spawn Python process + const pythonProcess = spawn('python3', args, { + cwd: repoRoot, + stdio: ['ignore', 'pipe', 'pipe'] + }); + + job.process = pythonProcess; + + let stdoutBuffer = ''; + let stderrBuffer = ''; + let finalResult: any = null; + + // Parse stdout for progress events and final JSON result + pythonProcess.stdout.on('data', (data) => { + stdoutBuffer += data.toString(); + const lines = stdoutBuffer.split('\n'); + stdoutBuffer = lines.pop() || ''; + + for (const line of lines) { + if (!line.trim()) continue; + try { + const parsed = JSON.parse(line); + if (parsed.type === 'progress') { + job.events.push(parsed); + } else if (parsed.errors !== undefined) { + finalResult = parsed; + } + } catch (e) { + // Line is not JSON + } + } + }); + + pythonProcess.stderr.on('data', (data) => { + stderrBuffer += data.toString(); + }); + + // Handle process errors + pythonProcess.on('error', (error) => { + console.error('[ValidationTrigger] Process error:', error); + job.status = 'error'; + job.events.push({ + type: 'error', + message: `Failed to spawn validation process: ${error.message}` + }); + job.endTime = Date.now(); + releaseValidationLock(); + }); + + // Handle process completion + pythonProcess.on('close', (code) => { + if (stdoutBuffer.trim()) { + try { + finalResult = JSON.parse(stdoutBuffer.trim()); + } catch (e) { + console.error('[ValidationTrigger] Failed to parse final result:', e); + } + } + + if (code === 0 || code === 1) { + job.status = 'complete'; + job.result = finalResult; + job.events.push({ + type: 'complete', + result: finalResult + }); + console.log('[ValidationTrigger] Background validation completed'); + } else { + job.status = 'error'; + job.events.push({ + type: 'error', + message: `Validation process exited with code ${code}`, + stderr: stderrBuffer + }); + console.error('[ValidationTrigger] Background validation failed:', stderrBuffer); + } + job.endTime = Date.now(); + + // Release the validation lock when job completes + releaseValidationLock(); + }); + + return true; +} diff --git a/webui/src/lib/stores/validationStore.ts b/webui/src/lib/stores/validationStore.ts new file mode 100644 index 0000000000..8d7ecd8a91 --- /dev/null +++ b/webui/src/lib/stores/validationStore.ts @@ -0,0 +1,74 @@ +import { writable, derived } from 'svelte/store'; + +export interface ValidationError { + level: 'ERROR' | 'WARNING'; + category: string; + message: string; + path: string | null; +} + +export interface ValidationResult { + errors: ValidationError[]; + error_count: number; + warning_count: number; + is_valid: boolean; +} + +interface ValidationState { + isValidating: boolean; + lastValidation: Date | null; + results: ValidationResult | null; +} + +function createValidationStore() { + const { subscribe, set, update } = writable({ + isValidating: false, + lastValidation: null, + results: null + }); + + return { + subscribe, + setValidating: (isValidating: boolean) => + update((state) => ({ + ...state, + isValidating + })), + setResults: (results: ValidationResult) => + update((state) => ({ + ...state, + isValidating: false, + lastValidation: new Date(), + results + })), + clear: () => + set({ + isValidating: false, + lastValidation: null, + results: null + }) + }; +} + +export const validationStore = createValidationStore(); + +// Derived stores for convenience +export const errorCount = derived(validationStore, ($store) => $store.results?.error_count ?? 0); + +export const warningCount = derived( + validationStore, + ($store) => $store.results?.warning_count ?? 0 +); + +export const errorsByCategory = derived(validationStore, ($store) => { + if (!$store.results) return new Map(); + + const grouped = new Map(); + for (const error of $store.results.errors) { + if (!grouped.has(error.category)) { + grouped.set(error.category, []); + } + grouped.get(error.category)!.push(error); + } + return grouped; +}); diff --git a/webui/src/lib/utils/pathToRoute.ts b/webui/src/lib/utils/pathToRoute.ts new file mode 100644 index 0000000000..2afb443acf --- /dev/null +++ b/webui/src/lib/utils/pathToRoute.ts @@ -0,0 +1,48 @@ +/** + * Convert file paths from validation errors to SvelteKit routes + * + * Example paths: + * - data/bambu_lab/PLA/basic_pla/white/variant.json -> /Brand/bambu_lab/PLA/basic_pla/white + * - stores/amazon/store.json -> /Store/amazon + */ +export function pathToRoute(filePath: string): string { + // Normalize path separators + const normalizedPath = filePath.replace(/\\/g, '/'); + const parts = normalizedPath.split('/'); + + // Handle stores directory + if (normalizedPath.startsWith('stores/')) { + // stores/store_name/store.json -> /Store/store_name + if (parts.length >= 2) { + return `/Store/${parts[1]}`; + } + } + + // Handle data directory hierarchy + if (normalizedPath.startsWith('data/')) { + const [_, brandId, materialId, filamentId, variantId, filename] = parts; + + // Brand level (data/brandId/brand.json) + if (filename === 'brand.json' || parts.length === 3) { + return `/Brand/${brandId}`; + } + + // Material level (data/brandId/materialId/material.json) + if (filename === 'material.json' || parts.length === 4) { + return `/Brand/${brandId}/${materialId}`; + } + + // Filament level (data/brandId/materialId/filamentId/filament.json) + if (filename === 'filament.json' || parts.length === 5) { + return `/Brand/${brandId}/${materialId}/${filamentId}`; + } + + // Variant level (data/brandId/materialId/filamentId/variantId/variant.json or sizes.json) + if (filename === 'variant.json' || filename === 'sizes.json' || parts.length === 6) { + return `/Brand/${brandId}/${materialId}/${filamentId}/${variantId}`; + } + } + + // Fallback to home if path doesn't match expected structure + return '/'; +} diff --git a/webui/src/routes/+layout.svelte b/webui/src/routes/+layout.svelte index 9e5f3b0ef7..dac138f37b 100644 --- a/webui/src/routes/+layout.svelte +++ b/webui/src/routes/+layout.svelte @@ -1,6 +1,8 @@