From a04794916304931655e93ac4f71f5c80aeccb137 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Fri, 12 Dec 2025 22:05:45 -0500 Subject: [PATCH 001/606] Add engine selection controls for clustered playlists --- clustered_playlists.py | 9 ++++- controllers/cluster_controller.py | 2 + main_gui.py | 63 ++++++++++++++++++++++++++----- 3 files changed, 62 insertions(+), 12 deletions(-) diff --git a/clustered_playlists.py b/clustered_playlists.py index 4a7512c..b3e5193 100644 --- a/clustered_playlists.py +++ b/clustered_playlists.py @@ -79,7 +79,12 @@ def cluster_tracks( def generate_clustered_playlists( - tracks, root_path: str, method: str, params: dict, log_callback=None + tracks, + root_path: str, + method: str, + params: dict, + log_callback=None, + engine: str = "librosa", ) -> None: """Create clustered playlists for the given tracks.""" if log_callback is None: @@ -99,7 +104,7 @@ def generate_clustered_playlists( cache = {} log_callback("→ No feature cache found; extracting all tracks") - log_callback("⚙ Extracting audio features with librosa…") + log_callback(f"⚙ Extracting audio features with {engine}…") feats = [] updated = False diff --git a/controllers/cluster_controller.py b/controllers/cluster_controller.py index 6b12208..8733291 100644 --- a/controllers/cluster_controller.py +++ b/controllers/cluster_controller.py @@ -48,6 +48,7 @@ def cluster_library( cluster_params: dict, log_callback, folder_filter: dict | None = None, + engine: str = "librosa", ) -> tuple[list[str], list]: """Generate clustered playlists for ``library_path`` and return features.""" @@ -69,5 +70,6 @@ def log(msg: str) -> None: method, cluster_params, log, + engine=engine, ) return tracks, feats diff --git a/main_gui.py b/main_gui.py index 2c8b83b..15de062 100644 --- a/main_gui.py +++ b/main_gui.py @@ -145,7 +145,10 @@ def km_func(X, p): n_clusters = 5 if cluster_cfg and cluster_cfg.get("method") == "kmeans": n_clusters = int(cluster_cfg.get("num", 5)) - params = {"n_clusters": n_clusters, "method": "kmeans"} + engine = "librosa" + if cluster_cfg and "engine" in cluster_cfg: + engine = cluster_cfg["engine"] + params = {"n_clusters": n_clusters, "method": "kmeans", "engine": engine} elif name == "Interactive – HDBSCAN": from hdbscan import HDBSCAN @@ -167,7 +170,15 @@ def km_func(X, p): extras["cluster_selection_epsilon"] = float( cluster_cfg["cluster_selection_epsilon"] ) - params = {"min_cluster_size": min_cs, "method": "hdbscan", **extras} + engine = "librosa" + if cluster_cfg and "engine" in cluster_cfg: + engine = cluster_cfg["engine"] + params = { + "min_cluster_size": min_cs, + "method": "hdbscan", + "engine": engine, + **extras, + } elif name == "Tempo/Energy Buckets": def _run(): path = app.require_library() @@ -304,12 +315,17 @@ def generate(): def _auto_create_all(): method = panel.cluster_params.get("method") - params = {k: v for k, v in panel.cluster_params.items() if k != "method"} + params = { + k: v + for k, v in panel.cluster_params.items() + if k not in {"method", "engine"} + } + engine = panel.cluster_params.get("engine", "librosa") if not params: return threading.Thread( target=app._run_cluster_generation, - args=(app.library_path, method, params), + args=(app.library_path, method, params, engine), daemon=True, ).start() @@ -1521,6 +1537,7 @@ def cluster_playlists_dialog(self, method: str = "kmeans"): dlg.resizable(True, True) method_var = tk.StringVar(value=method) + engine_var = tk.StringVar(value="") top = ttk.Frame(dlg) top.pack(fill="x", padx=10, pady=(10, 0)) @@ -1553,6 +1570,23 @@ def _update_fields(*args): params_frame = ttk.Frame(dlg) params_frame.pack(fill="x", padx=10, pady=(5, 0)) + engine_frame = ttk.LabelFrame(dlg, text="Feature Extraction Engine") + engine_frame.pack(fill="x", padx=10, pady=(10, 0)) + + ttk.Radiobutton( + engine_frame, + text="Librosa (current)", + variable=engine_var, + value="librosa", + ).pack(anchor="w", padx=5, pady=(2, 0)) + ttk.Radiobutton( + engine_frame, + text="Essentia (coming soon)", + variable=engine_var, + value="essentia", + state="disabled", + ).pack(anchor="w", padx=5, pady=(0, 5)) + # KMeans params km_frame = ttk.Frame(params_frame) km_var = tk.StringVar(value="5") @@ -1662,6 +1696,13 @@ def generate(): "exclude": list(exc_list.get(0, "end")), } + engine = engine_var.get() + if not engine: + messagebox.showerror( + "Select Engine", "Please select a feature extraction engine." + ) + return + m = method_var.get() params = {} if m == "kmeans": @@ -1698,12 +1739,12 @@ def generate(): f"{epsilon_var.get()} is not a valid number", ) return - self._start_cluster_playlists(m, params, dlg) + self._start_cluster_playlists(m, params, engine, dlg) ttk.Button(btns, text="Generate", command=generate).pack(side="left", padx=5) ttk.Button(btns, text="Cancel", command=dlg.destroy).pack(side="left", padx=5) - def _start_cluster_playlists(self, method: str, params: dict, dlg): + def _start_cluster_playlists(self, method: str, params: dict, engine: str, dlg): if dlg is not None: dlg.destroy() path = self.require_library() @@ -1711,16 +1752,18 @@ def _start_cluster_playlists(self, method: str, params: dict, dlg): return threading.Thread( target=self._run_cluster_generation, - args=(path, method, params), + args=(path, method, params, engine), daemon=True, ).start() - def _run_cluster_generation(self, path: str, method: str, params: dict): + def _run_cluster_generation( + self, path: str, method: str, params: dict, engine: str + ): tracks, feats = cluster_library( - path, method, params, self._log, self.folder_filter + path, method, params, self._log, self.folder_filter, engine ) self.cluster_data = (tracks, feats) - self.cluster_params = {"method": method, **params} + self.cluster_params = {"method": method, "engine": engine, **params} def done(): messagebox.showinfo("Clustered Playlists", "Generation complete") From 05001321e5c40317243d15e8cbbb4edcb2af8301 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sat, 13 Dec 2025 08:17:51 -0500 Subject: [PATCH 002/606] Fix crash log view indentation --- main_gui.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/main_gui.py b/main_gui.py index e7eda50..f353e59 100644 --- a/main_gui.py +++ b/main_gui.py @@ -2676,10 +2676,10 @@ def _view_crash_log(self) -> None: win = tk.Toplevel(self) win.title("Crash Log") - text = ScrolledText(win, width=80, height=24) - text.pack(fill="both", expand=True) - text.insert("end", "".join(lines)) - text.configure(state="disabled") + text = ScrolledText(win, width=80, height=24) + text.pack(fill="both", expand=True) + text.insert("end", "".join(lines)) + text.configure(state="disabled") def show_log_tab(self) -> None: """Switch to the Log tab so users can see background activity.""" From ec34dcb8b4daeb1377334ca4bec65a8d5bdba6e8 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sat, 13 Dec 2025 21:41:18 -0500 Subject: [PATCH 003/606] Add tempo energy bucket UI enhancements --- main_gui.py | 206 +++++++++++++++++++++++++++++++++++++++++++-- playlist_engine.py | 31 ++++++- 2 files changed, 226 insertions(+), 11 deletions(-) diff --git a/main_gui.py b/main_gui.py index f353e59..c79f96c 100644 --- a/main_gui.py +++ b/main_gui.py @@ -2,6 +2,7 @@ import threading import sys import logging +import webbrowser if sys.platform == "win32": try: @@ -78,6 +79,7 @@ from plugins.assistant_plugin import AssistantPlugin from controllers.cluster_controller import cluster_library from config import load_config, save_config, DEFAULT_FP_THRESHOLDS +import playlist_engine from playlist_engine import bucket_by_tempo_energy, more_like_this, autodj_playlist from controllers.cluster_controller import gather_tracks @@ -180,19 +182,207 @@ def km_func(X, p): **extras, } elif name == "Tempo/Energy Buckets": - def _run(): + lib_var = tk.StringVar(value=app.library_path or "No library selected") + dep_status = tk.StringVar() + progress_var = tk.StringVar(value="Waiting to start") + running = tk.BooleanVar(value=False) + total_tracks = 0 + q: queue.Queue = queue.Queue() + cancel_event = threading.Event() + + def open_path(path: str) -> None: + if not path: + return + try: + if sys.platform == "win32": + os.startfile(path) # type: ignore[attr-defined] + elif sys.platform == "darwin": + subprocess.run(["open", path], check=False) + else: + subprocess.run(["xdg-open", path], check=False) + except Exception as exc: # pragma: no cover - OS interaction + messagebox.showerror("Open File", f"Could not open {path}: {exc}") + + def open_instructions(_evt=None): + readme = os.path.abspath("README.md") + webbrowser.open_new(f"file://{readme}") + + def dependencies_ok() -> bool: + missing = [] + if playlist_engine.np is None: + missing.append("numpy") + if playlist_engine.librosa is None: + missing.append("librosa") + if missing: + dep_status.set( + "Missing dependencies: " + + ", ".join(missing) + + ". Install with `pip install -r requirements.txt`." + ) + return False + dep_status.set("Dependencies ready (numpy, librosa)") + return True + + def append_log(msg: str) -> None: + log_box.configure(state="normal") + log_box.insert("end", msg + "\n") + log_box.see("end") + log_box.configure(state="disabled") + + def render_stats(stats: dict | None): + for child in stats_rows.winfo_children(): + child.destroy() + if not stats: + ttk.Label(stats_rows, text="No playlists generated yet.").pack( + anchor="w", padx=5, pady=5 + ) + return + header = ttk.Frame(stats_rows) + header.pack(fill="x", padx=5, pady=(0, 4)) + ttk.Label(header, text="Tempo", width=10).pack(side="left") + ttk.Label(header, text="Energy", width=10).pack(side="left") + ttk.Label(header, text="Tracks", width=8).pack(side="left") + ttk.Label(header, text="Playlist").pack(side="left", padx=(10, 0)) + for (tb, eb), info in sorted(stats.items()): + row = ttk.Frame(stats_rows) + row.pack(fill="x", padx=5, pady=2) + ttk.Label(row, text=tb.title(), width=10).pack(side="left") + ttk.Label(row, text=eb.title(), width=10).pack(side="left") + ttk.Label(row, text=str(info.get("count", 0)), width=8).pack(side="left") + ttk.Label(row, text=os.path.basename(info.get("playlist", ""))).pack( + side="left", padx=(10, 5) + ) + ttk.Button( + row, + text="Open", + command=lambda p=info.get("playlist", ""): open_path(p), + ).pack(side="left") + + def update_controls(): + lib_var.set(app.library_path or "No library selected") + ready = bool(app.library_path) and dependencies_ok() and not running.get() + run_btn.config(state="normal" if ready else "disabled") + cancel_btn.config(state="normal" if running.get() else "disabled") + + def start_run(): + nonlocal total_tracks + if running.get(): + return path = app.require_library() if not path: return - app.show_log_tab() - tracks = gather_tracks(path) + if not dependencies_ok(): + messagebox.showerror("Missing Dependencies", dep_status.get()) + return + tracks = gather_tracks(path, getattr(app, "folder_filter", None)) + if not tracks: + messagebox.showinfo("No Tracks", "No audio files found in the library.") + return + total_tracks = len(tracks) + cancel_event.clear() + running.set(True) + progress["value"] = 0 + progress["maximum"] = total_tracks + progress_var.set(f"0/{total_tracks} processed") + log_box.configure(state="normal") + log_box.delete("1.0", "end") + log_box.configure(state="disabled") + append_log(f"Found {total_tracks} tracks. Generating tempo/energy buckets…") + render_stats(None) + update_controls() + + def worker(): + try: + result = bucket_by_tempo_energy( + tracks, + path, + log_callback=lambda m: q.put(("log", m)), + progress_callback=lambda c: q.put(("progress", c)), + cancel_event=cancel_event, + ) + q.put(("done", result)) + except Exception as exc: # pragma: no cover - UI surface only + q.put(("error", str(exc))) + + threading.Thread(target=worker, daemon=True).start() + poll_queue() + + def poll_queue(): try: - bucket_by_tempo_energy(tracks, path, app._log) - messagebox.showinfo("Buckets", "Tempo/Energy playlists written") - except Exception as e: - messagebox.showerror("Error", str(e)) + while True: + tag, payload = q.get_nowait() + if tag == "log": + append_log(payload) + elif tag == "progress": + progress["value"] = payload + progress_var.set(f"{payload}/{total_tracks} processed") + elif tag == "done": + running.set(False) + render_stats(payload.get("buckets")) + status = ( + "Cancelled" + if payload.get("cancelled") + else "Completed" + ) + progress_var.set( + f"{status} – processed {payload.get('processed')} of {payload.get('total')} tracks" + ) + update_controls() + elif tag == "error": + running.set(False) + update_controls() + progress_var.set("Error during bucket generation") + messagebox.showerror("Bucket Error", payload) + except queue.Empty: + pass + if running.get(): + frame.after(200, poll_queue) + + info = ttk.LabelFrame(frame, text="Tempo/Energy Buckets") + info.pack(fill="x", padx=10, pady=(5, 10)) + ttk.Label(info, textvariable=lib_var).pack(anchor="w", padx=5, pady=(5, 0)) + ttk.Label(info, textvariable=dep_status, wraplength=360).pack( + anchor="w", padx=5, pady=2 + ) + link = ttk.Label( + info, + text="View installation instructions", + foreground="blue", + cursor="hand2", + ) + link.pack(anchor="w", padx=5, pady=(0, 5)) + link.bind("", open_instructions) + + controls = ttk.Frame(frame) + controls.pack(fill="x", padx=10) + run_btn = ttk.Button(controls, text="Generate Buckets", command=start_run) + run_btn.pack(side="left") + cancel_btn = ttk.Button( + controls, text="Cancel", command=cancel_event.set, state="disabled" + ) + cancel_btn.pack(side="left", padx=(5, 0)) + ttk.Button( + controls, + text="Refresh Library Path", + command=update_controls, + ).pack(side="right") + + progress = ttk.Progressbar(frame, mode="determinate") + progress.pack(fill="x", padx=10, pady=(10, 0)) + ttk.Label(frame, textvariable=progress_var).pack(anchor="w", padx=12) + + log_group = ttk.LabelFrame(frame, text="Run Log") + log_group.pack(fill="both", expand=False, padx=10, pady=(10, 0)) + log_box = ScrolledText(log_group, height=8, state="disabled") + log_box.pack(fill="both", expand=True, padx=5, pady=5) + + stats_group = ttk.LabelFrame(frame, text="Bucket Summary") + stats_group.pack(fill="both", expand=True, padx=10, pady=(10, 10)) + stats_rows = ttk.Frame(stats_group) + stats_rows.pack(fill="both", expand=True) - ttk.Button(frame, text="Generate Buckets", command=_run).pack(padx=10, pady=10) + update_controls() + render_stats(None) return frame elif name == "More Like This": sel = tk.StringVar() diff --git a/playlist_engine.py b/playlist_engine.py index 1440c39..20b853a 100644 --- a/playlist_engine.py +++ b/playlist_engine.py @@ -37,15 +37,29 @@ def compute_tempo_energy(path: str) -> tuple[float, float]: return tempo, rms -def bucket_by_tempo_energy(tracks: list[str], root_path: str, log_callback=None) -> dict: +def bucket_by_tempo_energy( + tracks: list[str], + root_path: str, + log_callback=None, + progress_callback=None, + cancel_event=None, +) -> dict: if log_callback is None: log_callback = lambda m: None + if progress_callback is None: + progress_callback = lambda _count: None if librosa is None or np is None: - raise RuntimeError("librosa/numpy required for bucket generation") + raise RuntimeError( + "Tempo/Energy buckets require numpy and librosa. Install with `pip install -r requirements.txt`." + ) playlists_dir = os.path.join(root_path, "Playlists") os.makedirs(playlists_dir, exist_ok=True) buckets: dict[tuple[str, str], list[str]] = {} + processed = 0 for path in tracks: + if cancel_event and cancel_event.is_set(): + log_callback("! Bucket generation cancelled by user.") + break try: tempo, rms = compute_tempo_energy(path) tb = categorize_tempo(tempo) @@ -54,13 +68,24 @@ def bucket_by_tempo_energy(tracks: list[str], root_path: str, log_callback=None) log_callback(f"• {os.path.basename(path)} → {tb}/{eb}") except Exception as e: log_callback(f"! Failed analysis for {path}: {e}") + processed += 1 + progress_callback(processed) out_paths = {} for (tb, eb), items in buckets.items(): outfile = os.path.join(playlists_dir, f"{tb}_{eb}.m3u") write_playlist(items, outfile) out_paths[(tb, eb)] = outfile log_callback(f"→ Wrote {outfile}") - return out_paths + stats = { + (tb, eb): {"count": len(items), "playlist": out_paths[(tb, eb)]} + for (tb, eb), items in buckets.items() + } + return { + "buckets": stats, + "processed": processed, + "total": len(tracks), + "cancelled": bool(cancel_event and cancel_event.is_set()), + } def _get_feat(path: str, cache: dict, log_callback): From 17375a0f5550642b713af2febcc615cc3b9e5b85 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sat, 13 Dec 2025 21:51:21 -0500 Subject: [PATCH 004/606] Remove More Like This playlist tool --- README.md | 4 ++-- docs/project_documentation.html | 15 +++++++-------- main_gui.py | 27 +-------------------------- 3 files changed, 10 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 8ec0776..207e439 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ python main_gui.py 3. **Fix Tags** via the AcoustID menu (now supports multiple metadata services) 4. **Generate Playlists** from your folder structure 5. **Clustered Playlists** (interactive K-Means/HDBSCAN) via the Tools ▸ Clustered Playlists menu -6. **Smart Playlist Engine** with tempo/energy buckets and “More Like This” suggestions +6. **Smart Playlist Engine** with tempo/energy buckets 7. **Auto‑DJ** mode builds seamless playlists starting from any song 8. **Tidal-dl Sync** can upgrade low-quality files to FLAC 9. **Library Duplicate Scan** finds duplicate tracks after you drop new songs directly into your library @@ -51,7 +51,7 @@ python main_gui.py ### Playlist generator feedback -When you start a playlist job (tempo/energy buckets, *More Like This*, Auto‑DJ, or auto‑creating clustered playlists), the app automatically switches to the **Log** tab. The tab shows timestamped messages from the playlist helpers (feature gathering, similarity calculations, and playlist writes) so you can see that background work is running without waiting for a popup. +When you start a playlist job (tempo/energy buckets, Auto‑DJ, or auto‑creating clustered playlists), the app automatically switches to the **Log** tab. The tab shows timestamped messages from the playlist helpers (feature gathering, similarity calculations, and playlist writes) so you can see that background work is running without waiting for a popup. Cluster generation writes progress into `_log.txt` inside your library so you can review the steps later. diff --git a/docs/project_documentation.html b/docs/project_documentation.html index c535967..2d2f307 100644 --- a/docs/project_documentation.html +++ b/docs/project_documentation.html @@ -584,12 +584,11 @@

2.8 Playlist Generator (playlist_generator.py)

2.9 Playlist Engine

Advanced playlist features now include:

-
    -
  • Tempo/Energy Buckets: group tracks by BPM and RMS energy.
  • -
  • Nearest-Neighbor Playlists: “More Like This” suggestions.
  • -
  • Auto-DJ: chain similar songs for seamless playback.
  • -
  • Clustering Approaches: K-Means and HDBSCAN based sets.
  • -
  • Wireframe: playlist_wireframe.png
  • +
      +
    • Tempo/Energy Buckets: group tracks by BPM and RMS energy.
    • +
    • Auto-DJ: chain similar songs for seamless playback.
    • +
    • Clustering Approaches: K-Means and HDBSCAN based sets.
    • +
    • Wireframe: playlist_wireframe.png

    2.y GUI Front-end (main_gui.py)

    @@ -615,8 +614,8 @@

    3. Current Progress

  • Near-Duplicate Detector: integrated into music_indexer_api.py to remove fuzzy duplicates.
  • Clustered Playlists: interactive K-Means and HDBSCAN clustering with a graph-based panel.
  • UI Enhancements: theme selector and automatic DPI scaling.
  • -
  • Playlist Engine: tempo/energy buckets, “More Like This” suggestions and Auto-DJ mode.
  • -
+
  • Playlist Engine: tempo/energy buckets and Auto-DJ mode.
  • +
    diff --git a/main_gui.py b/main_gui.py index c79f96c..a3bcc6d 100644 --- a/main_gui.py +++ b/main_gui.py @@ -80,7 +80,7 @@ from controllers.cluster_controller import cluster_library from config import load_config, save_config, DEFAULT_FP_THRESHOLDS import playlist_engine -from playlist_engine import bucket_by_tempo_energy, more_like_this, autodj_playlist +from playlist_engine import bucket_by_tempo_energy, autodj_playlist from controllers.cluster_controller import gather_tracks FilterFn = Callable[[FileRecord], bool] @@ -384,30 +384,6 @@ def poll_queue(): update_controls() render_stats(None) return frame - elif name == "More Like This": - sel = tk.StringVar() - - def browse(): - f = filedialog.askopenfilename() - if f: - sel.set(f) - - def generate(): - path = app.require_library() - if not path or not sel.get(): - messagebox.showerror("Error", "Select a library and track") - return - app.show_log_tab() - tracks = gather_tracks(path) - res = more_like_this(sel.get(), tracks, 10, log_callback=app._log) - outfile = os.path.join(path, "Playlists", "more_like_this.m3u") - write_playlist(res, outfile) - messagebox.showinfo("Playlist", f"Written to {outfile}") - - tk.Entry(frame, textvariable=sel, width=40).pack(side="left", padx=5, pady=5) - ttk.Button(frame, text="Browse", command=browse).pack(side="left") - ttk.Button(frame, text="Generate", command=generate).pack(side="left", padx=5) - return frame elif name == "Auto-DJ": sel = tk.StringVar() count_var = tk.StringVar(value="20") @@ -971,7 +947,6 @@ def build_ui(self): "Sort by Genre", "Tempo/Energy Buckets", "Metadata", - "More Like This", "Auto-DJ", ]: self.plugin_list.insert("end", name) From 475688fb4801c259f20a98c688af0aed4431f74c Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sat, 13 Dec 2025 21:57:11 -0500 Subject: [PATCH 005/606] Add sort-by-genre playlist tool --- main_gui.py | 177 ++++++++++++++++++++++++++++++++++ playlist_engine.py | 112 ++++++++++++++++++++- tests/test_playlist_engine.py | 53 +++++++++- 3 files changed, 340 insertions(+), 2 deletions(-) diff --git a/main_gui.py b/main_gui.py index a3bcc6d..573e73c 100644 --- a/main_gui.py +++ b/main_gui.py @@ -181,6 +181,183 @@ def km_func(X, p): "engine": engine, **extras, } + elif name == "Sort by Genre": + lib_var = tk.StringVar(value=app.library_path or "No library selected") + progress_var = tk.StringVar(value="Waiting to start") + playlists_dir = tk.StringVar(value="") + running = tk.BooleanVar(value=False) + total_tracks = 0 + q: queue.Queue = queue.Queue() + cancel_event = threading.Event() + + def append_log(msg: str) -> None: + log_box.configure(state="normal") + log_box.insert("end", msg + "\n") + log_box.see("end") + log_box.configure(state="disabled") + + def render_stats(stats: dict | None): + for child in stats_rows.winfo_children(): + child.destroy() + if not stats: + ttk.Label(stats_rows, text="No playlists generated yet.").pack( + anchor="w", padx=5, pady=5 + ) + return + header = ttk.Frame(stats_rows) + header.pack(fill="x", padx=5, pady=(0, 4)) + ttk.Label(header, text="Genre", width=30, anchor="w").pack( + side="left" + ) + ttk.Label(header, text="Tracks", width=8).pack(side="left") + ttk.Label(header, text="Playlist").pack(side="left", padx=(10, 0)) + for genre, info in sorted(stats.items(), key=lambda kv: kv[0].lower()): + row = ttk.Frame(stats_rows) + row.pack(fill="x", padx=5, pady=2) + ttk.Label(row, text=genre, width=30, anchor="w").pack(side="left") + ttk.Label(row, text=str(info.get("count", 0)), width=8).pack( + side="left" + ) + ttk.Label(row, text=os.path.basename(info.get("playlist", ""))).pack( + side="left", padx=(10, 5) + ) + ttk.Button( + row, + text="Open", + command=lambda p=info.get("playlist", ""): open_path(p), + ).pack(side="left") + + def update_controls(): + lib_var.set(app.library_path or "No library selected") + ready = bool(app.library_path) and not running.get() + run_btn.config(state="normal" if ready else "disabled") + cancel_btn.config(state="normal" if running.get() else "disabled") + open_btn.config( + state="normal" if playlists_dir.get() or app.library_path else "disabled" + ) + + def open_path(path: str) -> None: + if not path: + return + try: + if sys.platform == "win32": + os.startfile(path) # type: ignore[attr-defined] + elif sys.platform == "darwin": + subprocess.run(["open", path], check=False) + else: + subprocess.run(["xdg-open", path], check=False) + except Exception as exc: # pragma: no cover - OS interaction + messagebox.showerror("Open File", f"Could not open {path}: {exc}") + + def start_run(): + nonlocal total_tracks + if running.get(): + return + path = app.require_library() + if not path: + return + tracks = gather_tracks(path, getattr(app, "folder_filter", None)) + if not tracks: + messagebox.showinfo("No Tracks", "No audio files found in the library.") + return + total_tracks = len(tracks) + playlists_dir.set(os.path.join(path, "Playlists", "Genres")) + cancel_event.clear() + running.set(True) + progress["value"] = 0 + progress["maximum"] = total_tracks + progress_var.set(f"0/{total_tracks} processed") + log_box.configure(state="normal") + log_box.delete("1.0", "end") + log_box.configure(state="disabled") + append_log(f"Found {total_tracks} tracks. Sorting by genre…") + render_stats(None) + update_controls() + + def worker(): + try: + result = playlist_engine.sort_tracks_by_genre( + tracks, + path, + log_callback=lambda m: q.put(("log", m)), + progress_callback=lambda c: q.put(("progress", c)), + cancel_event=cancel_event, + ) + q.put(("done", result)) + except Exception as exc: # pragma: no cover - UI surface only + q.put(("error", str(exc))) + + threading.Thread(target=worker, daemon=True).start() + poll_queue() + + def poll_queue(): + try: + while True: + tag, payload = q.get_nowait() + if tag == "log": + append_log(payload) + elif tag == "progress": + progress["value"] = payload + progress_var.set(f"{payload}/{total_tracks} processed") + elif tag == "done": + running.set(False) + if payload: + render_stats(payload.get("genres")) + update_controls() + elif tag == "error": + running.set(False) + messagebox.showerror("Sort by Genre", payload) + update_controls() + elif tag == "cancelled": + running.set(False) + append_log("Sorting cancelled") + update_controls() + except queue.Empty: + pass + if running.get(): + frame.after(200, poll_queue) + + def cancel_run(): + cancel_event.set() + q.put(("cancelled", None)) + + def open_genre_folder(): + path = playlists_dir.get() or ( + os.path.join(app.library_path, "Playlists", "Genres") + if app.library_path + else "" + ) + if not path: + return + os.makedirs(path, exist_ok=True) + open_path(path) + + info = ttk.Frame(frame) + info.pack(fill="x", pady=(0, 6)) + ttk.Label(info, textvariable=lib_var).pack(side="left") + open_btn = ttk.Button(info, text="Open Genre Playlists", command=open_genre_folder) + open_btn.pack(side="right", padx=(5, 0)) + run_btn = ttk.Button(info, text="Sort", command=start_run) + run_btn.pack(side="right", padx=(5, 0)) + cancel_btn = ttk.Button(info, text="Cancel", command=cancel_run, state="disabled") + cancel_btn.pack(side="right") + + prog_frame = ttk.Frame(frame) + prog_frame.pack(fill="x", pady=(0, 6)) + progress = ttk.Progressbar(prog_frame, mode="determinate") + progress.pack(fill="x") + ttk.Label(prog_frame, textvariable=progress_var).pack(anchor="w") + + stats_rows = ttk.Frame(frame) + stats_rows.pack(fill="x", pady=(0, 6)) + render_stats(None) + + ttk.Label(frame, text="Activity Log:").pack(anchor="w") + log_box = ScrolledText(frame, height=10, wrap="word", state="disabled") + log_box.pack(fill="both", expand=True) + + update_controls() + elif name == "Tempo/Energy Buckets": lib_var = tk.StringVar(value=app.library_path or "No library selected") dep_status = tk.StringVar() diff --git a/playlist_engine.py b/playlist_engine.py index 20b853a..0dfc680 100644 --- a/playlist_engine.py +++ b/playlist_engine.py @@ -1,5 +1,28 @@ +import importlib.util import os import math +import re +from typing import Callable, Iterable + +_mutagen_available = importlib.util.find_spec("mutagen") is not None +if _mutagen_available: + from mutagen import File as MutagenFile + from mutagen.easyid3 import EasyID3 + from mutagen.flac import FLAC +else: # pragma: no cover - optional dependency + class _DummyAudio(dict): + def get(self, key, default=None): + return [] + + def MutagenFile(*_a, **_k): + return None + + class EasyID3(_DummyAudio): + pass + + class FLAC(_DummyAudio): + pass + try: import numpy as np # type: ignore except Exception: # pragma: no cover - numpy optional @@ -80,8 +103,95 @@ def bucket_by_tempo_energy( (tb, eb): {"count": len(items), "playlist": out_paths[(tb, eb)]} for (tb, eb), items in buckets.items() } + + +def _split_genres(genres: Iterable[str]) -> list[str]: + parts: list[str] = [] + for raw in genres: + for chunk in re.split(r"[;,/|\\]", raw): + cleaned = chunk.strip() + if cleaned: + parts.append(cleaned) + return parts + + +def extract_genres(path: str) -> list[str]: + ext = os.path.splitext(path)[1].lower() + tags: list[str] = [] + if ext == ".mp3": + audio = EasyID3(path) + tags = [t for t in audio.get("genre", []) if isinstance(t, str)] + elif ext == ".flac": + audio = FLAC(path) + tags = [t for t in audio.get("genre", []) if isinstance(t, str)] + else: + audio = MutagenFile(path, easy=True) + if audio and audio.tags: + tags = [t for t in audio.tags.get("genre", []) if isinstance(t, str)] + return _split_genres(tags) + + +def _sanitize_genre(genre: str, used: set[str]) -> str: + safe = re.sub(r"[^\w\- ]+", "_", genre).strip() or "Unknown" + candidate = safe.replace(" ", "_") + if candidate not in used: + used.add(candidate) + return candidate + counter = 2 + while f"{candidate}_{counter}" in used: + counter += 1 + final = f"{candidate}_{counter}" + used.add(final) + return final + + +def sort_tracks_by_genre( + tracks: list[str], + root_path: str, + log_callback=None, + progress_callback=None, + cancel_event=None, + genre_reader: Callable[[str], list[str]] | None = None, +) -> dict: + """Group tracks by genre and write one playlist per genre.""" + + if log_callback is None: + log_callback = lambda m: None + if progress_callback is None: + progress_callback = lambda _count: None + reader = genre_reader or extract_genres + + playlists_dir = os.path.join(root_path, "Playlists", "Genres") + os.makedirs(playlists_dir, exist_ok=True) + + buckets: dict[str, list[str]] = {} + processed = 0 + for path in tracks: + if cancel_event and cancel_event.is_set(): + log_callback("! Genre sorting cancelled by user.") + break + try: + genres = reader(path) or ["Unknown"] + except Exception as exc: + log_callback(f"! Failed to read genres from {path}: {exc}") + genres = ["Unknown"] + for g in genres: + buckets.setdefault(g, []).append(path) + log_callback(f"• {os.path.basename(path)} → {', '.join(genres)}") + processed += 1 + progress_callback(processed) + + used_names: set[str] = set() + stats = {} + for genre, items in sorted(buckets.items(), key=lambda kv: kv[0].lower()): + fname = _sanitize_genre(genre, used_names) + ".m3u" + out_path = os.path.join(playlists_dir, fname) + write_playlist(items, out_path) + stats[genre] = {"count": len(items), "playlist": out_path} + log_callback(f"→ Wrote {out_path}") + return { - "buckets": stats, + "genres": stats, "processed": processed, "total": len(tracks), "cancelled": bool(cancel_event and cancel_event.is_set()), diff --git a/tests/test_playlist_engine.py b/tests/test_playlist_engine.py index 79d8f27..3118fca 100644 --- a/tests/test_playlist_engine.py +++ b/tests/test_playlist_engine.py @@ -1,4 +1,12 @@ -from playlist_engine import categorize_tempo, categorize_energy, more_like_this, autodj_playlist +import os + +from playlist_engine import ( + categorize_tempo, + categorize_energy, + more_like_this, + autodj_playlist, + sort_tracks_by_genre, +) def test_categorize_helpers(): @@ -24,3 +32,46 @@ def test_more_like_this_and_autodj(): dj = autodj_playlist("a", tracks, n=3, feature_cache=feats) assert dj == ["a", "b", "c"] + + +def test_sort_tracks_by_genre(tmp_path): + rock = tmp_path / "rock" / "song1.mp3" + jazz = tmp_path / "jazz" / "song2.flac" + mixed = tmp_path / "var" / "song3.wav" + for p in [rock, jazz, mixed]: + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text("test") + + genre_map = { + str(rock): ["Rock"], + str(jazz): ["Jazz"], + str(mixed): ["Jazz", "Rock"], + } + + logs: list[str] = [] + progress: list[int] = [] + result = sort_tracks_by_genre( + [str(rock), str(jazz), str(mixed)], + str(tmp_path), + log_callback=lambda m: logs.append(m), + progress_callback=lambda c: progress.append(c), + genre_reader=lambda p: genre_map[p], + ) + + playlists_dir = tmp_path / "Playlists" / "Genres" + rock_playlist = (playlists_dir / "Rock.m3u").read_text().splitlines() + jazz_playlist = (playlists_dir / "Jazz.m3u").read_text().splitlines() + + assert result["genres"]["Rock"]["count"] == 2 + assert result["genres"]["Jazz"]["count"] == 2 + rock_rel = os.path.relpath(rock, playlists_dir) + jazz_rel = os.path.relpath(jazz, playlists_dir) + mixed_rel = os.path.relpath(mixed, playlists_dir) + + assert rock_rel in rock_playlist + assert mixed_rel in rock_playlist + assert jazz_rel in jazz_playlist + assert mixed_rel in jazz_playlist + assert progress == [1, 2, 3] + assert len(logs) == 5 + assert any("song1" in log for log in logs) From 4c5bf6968f79861a597ce86cddea0c832430f2fe Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sat, 13 Dec 2025 22:05:02 -0500 Subject: [PATCH 006/606] Trim redundant Tools menu actions --- main_gui.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/main_gui.py b/main_gui.py index 573e73c..e62b200 100644 --- a/main_gui.py +++ b/main_gui.py @@ -927,9 +927,6 @@ def build_ui(self): menubar.add_cascade(label="Settings", menu=settings_menu) tools_menu = tk.Menu(menubar, tearoff=False) - tools_menu.add_command( - label="Regenerate Playlists", command=self.regenerate_playlists - ) tools_menu.add_command(label="Fix Tags via AcoustID", command=self.fix_tags_gui) tools_menu.add_command( label="Generate Library Index…", @@ -952,16 +949,6 @@ def build_ui(self): label="Genre Normalizer", command=self._open_genre_normalizer ) tools_menu.add_command(label="Reset Tag-Fix Log", command=self.reset_tagfix_log) - cluster_menu = tk.Menu(tools_menu, tearoff=False) - cluster_menu.add_command( - label="K-Means…", - command=lambda: self.cluster_playlists_dialog("kmeans"), - ) - cluster_menu.add_command( - label="HDBSCAN…", - command=lambda: self.cluster_playlists_dialog("hdbscan"), - ) - tools_menu.add_cascade(label="Clustered Playlists", menu=cluster_menu) menubar.add_cascade(label="Tools", menu=tools_menu) debug_menu = tk.Menu(menubar, tearoff=False) @@ -1862,16 +1849,6 @@ def compare_libraries(self): self._log(f"[stub] Compare Libraries → Master: {master}, Device: {device}") self.update_library_info() - def regenerate_playlists(self): - path = self.require_library() - if not path: - return - messagebox.showinfo( - "Regenerate Playlists", f"[stub] Would regenerate playlists in:\n{path}" - ) - self._log(f"[stub] Regenerate Playlists → {path}") - self.update_library_info() - def cluster_playlists_dialog(self, method: str = "kmeans"): path = self.require_library() if not path: From f1090be9037300d0fd759ad0d625c3e04816a085 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sat, 13 Dec 2025 22:06:05 -0500 Subject: [PATCH 007/606] Add manual export controls to genre playlists --- main_gui.py | 117 ++++++++++++++++++++++++++++++---- playlist_engine.py | 95 +++++++++++++++++++++++++-- tests/test_playlist_engine.py | 38 +++++++++++ 3 files changed, 232 insertions(+), 18 deletions(-) diff --git a/main_gui.py b/main_gui.py index 573e73c..d4efa38 100644 --- a/main_gui.py +++ b/main_gui.py @@ -189,6 +189,9 @@ def km_func(X, p): total_tracks = 0 q: queue.Queue = queue.Queue() cancel_event = threading.Event() + genre_vars: dict[str, tk.BooleanVar] = {} + current_buckets: dict[str, list[str]] = {} + planned_paths: dict[str, str] = {} def append_log(msg: str) -> None: log_box.configure(state="normal") @@ -199,42 +202,66 @@ def append_log(msg: str) -> None: def render_stats(stats: dict | None): for child in stats_rows.winfo_children(): child.destroy() + genre_vars.clear() if not stats: - ttk.Label(stats_rows, text="No playlists generated yet.").pack( - anchor="w", padx=5, pady=5 - ) + ttk.Label( + stats_rows, + text="No playlists generated yet. Run a sort to preview genres.", + ).pack(anchor="w", padx=5, pady=5) + update_controls() return header = ttk.Frame(stats_rows) header.pack(fill="x", padx=5, pady=(0, 4)) - ttk.Label(header, text="Genre", width=30, anchor="w").pack( - side="left" + ttk.Label(header, text="Export", width=8, anchor="w").pack( + side="left", padx=(0, 4) ) + ttk.Label(header, text="Genre", width=26, anchor="w").pack(side="left") ttk.Label(header, text="Tracks", width=8).pack(side="left") - ttk.Label(header, text="Playlist").pack(side="left", padx=(10, 0)) + ttk.Label(header, text="Playlist", width=20, anchor="w").pack( + side="left", padx=(10, 0) + ) + ttk.Label(header, text="Status", width=12, anchor="w").pack( + side="left", padx=(6, 0) + ) for genre, info in sorted(stats.items(), key=lambda kv: kv[0].lower()): row = ttk.Frame(stats_rows) row.pack(fill="x", padx=5, pady=2) - ttk.Label(row, text=genre, width=30, anchor="w").pack(side="left") + var = tk.BooleanVar(value=info.get("exported", False) or True) + genre_vars[genre] = var + ttk.Checkbutton(row, variable=var).pack(side="left", padx=(0, 6)) + ttk.Label(row, text=genre, width=26, anchor="w").pack(side="left") ttk.Label(row, text=str(info.get("count", 0)), width=8).pack( side="left" ) - ttk.Label(row, text=os.path.basename(info.get("playlist", ""))).pack( - side="left", padx=(10, 5) - ) + ttk.Label( + row, text=os.path.basename(info.get("playlist", "")), width=20 + ).pack(side="left", padx=(10, 5)) + status_txt = "Exported" if info.get("exported") else "Pending" + ttk.Label(row, text=status_txt, width=12).pack(side="left", padx=(6, 4)) ttk.Button( row, text="Open", command=lambda p=info.get("playlist", ""): open_path(p), + state="normal" if info.get("exported") else "disabled", ).pack(side="left") + update_controls() def update_controls(): lib_var.set(app.library_path or "No library selected") ready = bool(app.library_path) and not running.get() + has_results = bool(current_buckets) run_btn.config(state="normal" if ready else "disabled") cancel_btn.config(state="normal" if running.get() else "disabled") open_btn.config( state="normal" if playlists_dir.get() or app.library_path else "disabled" ) + export_btn.config( + state="normal" + if ready and has_results and any(v.get() for v in genre_vars.values()) + else "disabled" + ) + select_all_btn.config(state="normal" if has_results and ready else "disabled") + select_none_btn.config(state="normal" if has_results and ready else "disabled") def open_path(path: str) -> None: if not path: @@ -249,6 +276,36 @@ def open_path(path: str) -> None: except Exception as exc: # pragma: no cover - OS interaction messagebox.showerror("Open File", f"Could not open {path}: {exc}") + def select_all(): + for var in genre_vars.values(): + var.set(True) + update_controls() + + def select_none(): + for var in genre_vars.values(): + var.set(False) + update_controls() + + def export_selected(): + if running.get() or not current_buckets: + return + selected = [g for g, var in genre_vars.items() if var.get()] + if not selected: + messagebox.showinfo( + "No Genres Selected", + "Select at least one genre to export playlists.", + ) + return + result = playlist_engine.export_genre_playlists( + current_buckets, + app.library_path, + selected_genres=selected, + log_callback=append_log, + planned_paths=planned_paths, + ) + render_stats(result.get("genres")) + append_log(f"Exported {len(selected)} playlist(s).") + def start_run(): nonlocal total_tracks if running.get(): @@ -256,6 +313,9 @@ def start_run(): path = app.require_library() if not path: return + current_buckets.clear() + planned_paths.clear() + genre_vars.clear() tracks = gather_tracks(path, getattr(app, "folder_filter", None)) if not tracks: messagebox.showinfo("No Tracks", "No audio files found in the library.") @@ -282,6 +342,7 @@ def worker(): log_callback=lambda m: q.put(("log", m)), progress_callback=lambda c: q.put(("progress", c)), cancel_event=cancel_event, + export=False, ) q.put(("done", result)) except Exception as exc: # pragma: no cover - UI surface only @@ -291,6 +352,7 @@ def worker(): poll_queue() def poll_queue(): + nonlocal current_buckets, planned_paths try: while True: tag, payload = q.get_nowait() @@ -302,6 +364,8 @@ def poll_queue(): elif tag == "done": running.set(False) if payload: + current_buckets = payload.get("buckets", {}) + planned_paths = payload.get("playlist_paths", {}) render_stats(payload.get("genres")) update_controls() elif tag == "error": @@ -348,8 +412,37 @@ def open_genre_folder(): progress.pack(fill="x") ttk.Label(prog_frame, textvariable=progress_var).pack(anchor="w") - stats_rows = ttk.Frame(frame) - stats_rows.pack(fill="x", pady=(0, 6)) + selection_bar = ttk.Frame(frame) + selection_bar.pack(fill="x", pady=(0, 4)) + select_all_btn = ttk.Button( + selection_bar, text="Select All", command=select_all, state="disabled" + ) + select_all_btn.pack(side="left") + select_none_btn = ttk.Button( + selection_bar, text="Select None", command=select_none, state="disabled" + ) + select_none_btn.pack(side="left", padx=(5, 0)) + export_btn = ttk.Button( + selection_bar, + text="Export Selected Playlists", + command=export_selected, + state="disabled", + ) + export_btn.pack(side="right") + + stats_container = ttk.Frame(frame) + stats_container.pack(fill="both", pady=(0, 6), expand=True) + stats_canvas = tk.Canvas(stats_container, height=220) + stats_scroll = ttk.Scrollbar(stats_container, orient="vertical", command=stats_canvas.yview) + stats_rows = ttk.Frame(stats_canvas) + stats_rows.bind( + "", + lambda e: stats_canvas.configure(scrollregion=stats_canvas.bbox("all")), + ) + stats_canvas.create_window((0, 0), window=stats_rows, anchor="nw") + stats_canvas.configure(yscrollcommand=stats_scroll.set) + stats_canvas.pack(side="left", fill="both", expand=True) + stats_scroll.pack(side="right", fill="y") render_stats(None) ttk.Label(frame, text="Activity Log:").pack(anchor="w") diff --git a/playlist_engine.py b/playlist_engine.py index 0dfc680..4028654 100644 --- a/playlist_engine.py +++ b/playlist_engine.py @@ -4,7 +4,10 @@ import re from typing import Callable, Iterable -_mutagen_available = importlib.util.find_spec("mutagen") is not None +try: + _mutagen_available = importlib.util.find_spec("mutagen") is not None +except ValueError: # pragma: no cover - defensive for broken environments + _mutagen_available = False if _mutagen_available: from mutagen import File as MutagenFile from mutagen.easyid3 import EasyID3 @@ -152,8 +155,15 @@ def sort_tracks_by_genre( progress_callback=None, cancel_event=None, genre_reader: Callable[[str], list[str]] | None = None, + export: bool = True, + selected_genres: set[str] | list[str] | None = None, ) -> dict: - """Group tracks by genre and write one playlist per genre.""" + """Group tracks by genre and optionally write playlists. + + If ``export`` is False, this function will only analyze genres and return + stats without writing any playlists. Pass ``selected_genres`` to export a + subset of genres when ``export`` is True. + """ if log_callback is None: log_callback = lambda m: None @@ -162,7 +172,8 @@ def sort_tracks_by_genre( reader = genre_reader or extract_genres playlists_dir = os.path.join(root_path, "Playlists", "Genres") - os.makedirs(playlists_dir, exist_ok=True) + if export: + os.makedirs(playlists_dir, exist_ok=True) buckets: dict[str, list[str]] = {} processed = 0 @@ -183,21 +194,93 @@ def sort_tracks_by_genre( used_names: set[str] = set() stats = {} + planned_paths: dict[str, str] = {} + selected = set(selected_genres) if selected_genres else None for genre, items in sorted(buckets.items(), key=lambda kv: kv[0].lower()): fname = _sanitize_genre(genre, used_names) + ".m3u" out_path = os.path.join(playlists_dir, fname) - write_playlist(items, out_path) - stats[genre] = {"count": len(items), "playlist": out_path} - log_callback(f"→ Wrote {out_path}") + planned_paths[genre] = out_path + should_export = export and (selected is None or genre in selected) + if should_export: + write_playlist(items, out_path) + log_callback(f"→ Wrote {out_path}") + stats[genre] = { + "count": len(items), + "playlist": out_path, + "exported": should_export, + } + if export and not should_export: + log_callback(f"• Skipped exporting {genre}") return { "genres": stats, "processed": processed, "total": len(tracks), "cancelled": bool(cancel_event and cancel_event.is_set()), + "buckets": buckets, + "playlist_paths": planned_paths, } +def export_genre_playlists( + buckets: dict[str, list[str]], + root_path: str, + selected_genres: set[str] | list[str] | None = None, + log_callback=None, + planned_paths: dict[str, str] | None = None, +): + """Write playlists for the provided genre buckets. + + Parameters + ---------- + buckets : dict + Mapping of genre -> list of track paths. + root_path : str + Root music library path used to resolve playlist directory. + selected_genres : set | list | None + Optional subset of genres to export. If None, all genres are exported. + log_callback : callable | None + Callback for writing log messages. + planned_paths : dict | None + Optional mapping of genre -> playlist path generated earlier. When + provided, the same filenames are reused to keep the preview stable. + """ + + if log_callback is None: + log_callback = lambda m: None + + playlists_dir = os.path.join(root_path, "Playlists", "Genres") + os.makedirs(playlists_dir, exist_ok=True) + + selected = set(selected_genres) if selected_genres else None + used_names: set[str] = set() + stats: dict[str, dict] = {} + + for genre, items in sorted(buckets.items(), key=lambda kv: kv[0].lower()): + if planned_paths and genre in planned_paths: + out_path = planned_paths[genre] + # Ensure later genres do not reuse the same base name + used_names.add(os.path.splitext(os.path.basename(out_path))[0]) + else: + fname = _sanitize_genre(genre, used_names) + ".m3u" + out_path = os.path.join(playlists_dir, fname) + + should_export = selected is None or genre in selected + if should_export: + write_playlist(items, out_path) + log_callback(f"→ Wrote {out_path}") + else: + log_callback(f"• Skipped exporting {genre}") + + stats[genre] = { + "count": len(items), + "playlist": out_path, + "exported": should_export, + } + + return {"genres": stats, "playlists_dir": playlists_dir} + + def _get_feat(path: str, cache: dict, log_callback): if path not in cache: from clustered_playlists import extract_audio_features diff --git a/tests/test_playlist_engine.py b/tests/test_playlist_engine.py index 3118fca..8d20870 100644 --- a/tests/test_playlist_engine.py +++ b/tests/test_playlist_engine.py @@ -5,6 +5,7 @@ categorize_energy, more_like_this, autodj_playlist, + export_genre_playlists, sort_tracks_by_genre, ) @@ -75,3 +76,40 @@ def test_sort_tracks_by_genre(tmp_path): assert progress == [1, 2, 3] assert len(logs) == 5 assert any("song1" in log for log in logs) + + +def test_manual_genre_export(tmp_path): + rock = tmp_path / "rock" / "song1.mp3" + jazz = tmp_path / "jazz" / "song2.flac" + for p in [rock, jazz]: + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text("test") + + genre_map = {str(rock): ["Rock"], str(jazz): ["Jazz"]} + + result = sort_tracks_by_genre( + [str(rock), str(jazz)], + str(tmp_path), + log_callback=lambda _m: None, + genre_reader=lambda p: genre_map[p], + export=False, + ) + + playlists_dir = tmp_path / "Playlists" / "Genres" + assert not playlists_dir.exists() + assert "buckets" in result and "playlist_paths" in result + + logs: list[str] = [] + exported = export_genre_playlists( + result["buckets"], + str(tmp_path), + selected_genres={"Jazz"}, + log_callback=lambda m: logs.append(m), + planned_paths=result["playlist_paths"], + ) + + assert (playlists_dir / "Jazz.m3u").exists() + assert not (playlists_dir / "Rock.m3u").exists() + assert exported["genres"]["Jazz"]["exported"] is True + assert exported["genres"]["Rock"]["exported"] is False + assert any("Jazz.m3u" in log for log in logs) From 8018d22bf9b89739823aac71e5cd8f97e975440c Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sat, 13 Dec 2025 22:08:05 -0500 Subject: [PATCH 008/606] Remove redundant File menu actions --- main_gui.py | 129 ---------------------------------------------------- 1 file changed, 129 deletions(-) diff --git a/main_gui.py b/main_gui.py index 573e73c..04999d1 100644 --- a/main_gui.py +++ b/main_gui.py @@ -43,7 +43,6 @@ import fingerprint_cache import chromaprint_utils from controllers.library_index_controller import generate_index -from controllers.import_controller import import_new_files from controllers.genre_list_controller import list_unique_genres from controllers.highlight_controller import play_snippet, PYDUB_AVAILABLE from controllers.scan_progress_controller import ScanProgressController @@ -912,10 +911,6 @@ def build_ui(self): file_menu = tk.Menu(menubar, tearoff=False) file_menu.add_command(label="Open Library…", command=self.select_library) file_menu.add_command(label="Validate Library", command=self.validate_library) - file_menu.add_command(label="Import New Songs", command=self.import_songs) - file_menu.add_command(label="Scan for Orphans", command=self.scan_orphans) - file_menu.add_command(label="Compare Libraries", command=self.compare_libraries) - file_menu.add_command(label="Show All Files", command=self._on_show_all) file_menu.add_separator() file_menu.add_command(label="Exit", command=self._on_exit) menubar.add_cascade(label="File", menu=file_menu) @@ -1478,95 +1473,6 @@ def validate_library(self): self._log(f"✘ Invalid SoundVault: {path}\n" + "\n".join(errors)) self.update_library_info() - def import_songs(self): - initial = load_last_path() - vault = filedialog.askdirectory( - title="Select SoundVault Root", initialdir=initial - ) - if not vault: - return - - save_last_path(vault) - - is_valid, errors = validate_soundvault_structure(vault) - if not is_valid: - messagebox.showerror("Invalid SoundVault", "\n".join(errors)) - self._log(f"✘ Invalid SoundVault: {vault}\n" + "\n".join(errors)) - return - - import_folder = filedialog.askdirectory( - title="Select Folder of New Songs", initialdir=vault - ) - if not import_folder: - return - - dry_run = messagebox.askyesno("Dry Run?", "Perform a dry-run preview only?") - estimate = messagebox.askyesno( - "Estimate BPM?", "Attempt BPM estimation for missing values?" - ) - - stop_on_error = False - if not dry_run: - stop_on_error = messagebox.askyesno( - "Stop on Error?", - "Stop the import if any file move fails?", - ) - - def log_line(msg: str) -> None: - self.after(0, lambda m=msg: self._log(m)) - - def task() -> None: - try: - summary = import_new_files( - vault, - import_folder, - dry_run=dry_run, - estimate_bpm=estimate, - log_callback=log_line, - stop_on_error=stop_on_error, - ) - - def ui_complete() -> None: - if summary["dry_run"]: - messagebox.showinfo( - "Dry Run Complete", - f"Preview written to:\n{summary['html']}", - ) - else: - moved = summary.get("moved", 0) - base_msg = ( - f"Imported {moved} files. Preview:\n{summary['html']}" - ) - if summary.get("errors"): - err_text = "\n".join(summary["errors"]) - full_msg = f"{base_msg}\n\nErrors:\n{err_text}" - if messagebox.askretrycancel( - "Import Complete (with errors)", full_msg - ): - threading.Thread(target=task, daemon=True).start() - return - else: - messagebox.showinfo("Import Complete", base_msg) - - if summary.get("errors"): - for err in summary["errors"]: - self._log(f"! {err}") - - self._log( - f"✓ Import finished for {import_folder} → {vault}. Dry run: {dry_run}. BPM: {estimate}." - ) - - self.after(0, ui_complete) - except Exception as e: - - def ui_err() -> None: - messagebox.showerror("Import failed", str(e)) - self._log(f"✘ Import failed for {import_folder}: {e}") - - self.after(0, ui_err) - - threading.Thread(target=task, daemon=True).start() - # ── Quality Checker Actions ──────────────────────────────────────── def scan_duplicates(self): folder = self.library_path_var.get() @@ -1835,33 +1741,6 @@ def _confirm_duplicates(self, dups): top.wait_window() return proceed.get() - def scan_orphans(self): - path = self.require_library() - if not path: - return - messagebox.showinfo( - "Scan for Orphans", f"[stub] Would scan for orphans in:\n{path}" - ) - self._log(f"[stub] Scan for Orphans → {path}") - self.update_library_info() - - def compare_libraries(self): - master = self.require_library() - if not master: - return - device = filedialog.askdirectory( - title="Select Device Library Root", initialdir=master - ) - if not device: - return - save_last_path(device) - messagebox.showinfo( - "Compare Libraries", - f"[stub] Would compare:\nMaster: {master}\nDevice: {device}", - ) - self._log(f"[stub] Compare Libraries → Master: {master}, Device: {device}") - self.update_library_info() - def regenerate_playlists(self): path = self.require_library() if not path: @@ -2175,14 +2054,6 @@ def on_show_all(): dlg.wait_window() return result["proceed"], var_no_diff.get(), var_skipped.get(), show_all - def _on_show_all(self): - """Run tag-fix scan showing every file regardless of prior log.""" - self.tagfix_show_all.set(True) - try: - self.fix_tags_gui() - finally: - self.tagfix_show_all.set(False) - def fix_tags_gui(self): folder = self.tagfix_folder_var.get() if not folder: From 7b4c00ce90042a0c27c58aaf72cca3e94190ddad Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sat, 13 Dec 2025 22:17:37 -0500 Subject: [PATCH 009/606] Remove play highlight menu option --- main_gui.py | 31 +------------------------------ 1 file changed, 1 insertion(+), 30 deletions(-) diff --git a/main_gui.py b/main_gui.py index 925eaac..9d3335a 100644 --- a/main_gui.py +++ b/main_gui.py @@ -44,7 +44,7 @@ import chromaprint_utils from controllers.library_index_controller import generate_index from controllers.genre_list_controller import list_unique_genres -from controllers.highlight_controller import play_snippet, PYDUB_AVAILABLE +from controllers.highlight_controller import PYDUB_AVAILABLE from controllers.scan_progress_controller import ScanProgressController from gui.audio_preview import PreviewPlayer from io import BytesIO @@ -1024,14 +1024,6 @@ def build_ui(self): label="List Unique Genres…", command=lambda: list_unique_genres(self.require_library()), ) - if PYDUB_AVAILABLE and self.ffmpeg_available: - tools_menu.add_command( - label="Play Highlight…", command=self.sample_song_highlight - ) - else: - tools_menu.add_command( - label="Play Highlight… (requires pydub & ffmpeg)", state="disabled" - ) tools_menu.add_separator() tools_menu.add_command( label="Genre Normalizer", command=self._open_genre_normalizer @@ -2460,27 +2452,6 @@ def _render_table(self, records: List[FileRecord]): tags=(tag,), ) - def sample_song_highlight(self): - """Ask the user for an audio file and play its highlight.""" - initial = load_last_path() - path = filedialog.askopenfilename( - title="Select Audio File", - initialdir=initial, - filetypes=[("Audio Files", "*.mp3 *.wav *.flac *.ogg"), ("All files", "*")], - ) - if not path: - return - - save_last_path(os.path.dirname(path)) - try: - start_sec = play_snippet(path) - self._log( - f"Played highlight of '{os.path.basename(path)}' starting at {start_sec:.2f}s" - ) - except Exception as e: - messagebox.showerror("Playback failed", str(e)) - self._log(f"✘ Playback failed for {path}: {e}") - def _open_tagfix_debug_window(self): if ( getattr(self, "tagfix_debug_win", None) From 78dd029580913d9b278a06876c9d764c398051dd Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sat, 13 Dec 2025 22:24:17 -0500 Subject: [PATCH 010/606] Add player tab for 30s track previews --- main_gui.py | 191 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 187 insertions(+), 4 deletions(-) diff --git a/main_gui.py b/main_gui.py index 9d3335a..74d108a 100644 --- a/main_gui.py +++ b/main_gui.py @@ -971,6 +971,12 @@ def __init__(self): on_done=lambda: self.after(0, self._preview_finished_ui) ) + # Player tab state + self.player_tracks: list[dict[str, str]] = [] + self.player_tree_paths: dict[str, str] = {} + self.player_status_var = tk.StringVar(value="Select a library to load tracks.") + self._player_load_thread: threading.Thread | None = None + # assume ffmpeg is available without performing checks self.ffmpeg_available = True @@ -1367,6 +1373,54 @@ def build_ui(self): self.scan_btn.config(state="normal") self._validate_threshold() + # ─── Player Tab ──────────────────────────────────────────────── + self.player_tab = ttk.Frame(self.notebook) + self.notebook.add(self.player_tab, text="Player") + + player_controls = ttk.Frame(self.player_tab) + player_controls.pack(fill="x", padx=10, pady=(10, 5)) + ttk.Label(player_controls, textvariable=self.player_status_var).pack( + side="left" + ) + self.player_reload_btn = ttk.Button( + player_controls, + text="Reload", + command=self._load_player_library_async, + state="disabled", + ) + self.player_reload_btn.pack(side="right") + + player_table = ttk.Frame(self.player_tab) + player_table.pack(fill="both", expand=True, padx=10, pady=(0, 10)) + p_vsb = ttk.Scrollbar(player_table, orient="vertical") + p_vsb.pack(side="right", fill="y") + p_hsb = ttk.Scrollbar(player_table, orient="horizontal") + p_hsb.pack(side="bottom", fill="x") + + cols = ("Title", "Artist", "Album", "Length", "Play") + self.player_tree = ttk.Treeview( + player_table, + columns=cols, + show="headings", + yscrollcommand=p_vsb.set, + xscrollcommand=p_hsb.set, + ) + p_vsb.config(command=self.player_tree.yview) + p_hsb.config(command=self.player_tree.xview) + self.player_tree.pack(fill="both", expand=True) + + widths = {"Title": 220, "Artist": 140, "Album": 160, "Length": 70, "Play": 50} + for c in cols: + self.player_tree.heading(c, text=c) + self.player_tree.column( + c, + width=widths.get(c, 100), + anchor="center" if c in {"Length", "Play"} else "w", + stretch=c != "Play", + ) + + self.player_tree.bind("", self._on_player_tree_click) + # ─── Library Sync Tab ───────────────────────────────────────────── self.sync_tab = ttk.Frame(self.notebook) self.notebook.add(self.sync_tab, text="Library Sync") @@ -1510,10 +1564,16 @@ def select_library(self): self.scan_btn.config(state="normal") self._validate_threshold() self.update_library_info() + if hasattr(self, "player_reload_btn"): + self.player_reload_btn.config(state="normal") + self._load_player_library_async() def update_library_info(self): if not self.library_path: self.library_stats_var.set("") + if hasattr(self, "player_reload_btn"): + self.player_reload_btn.config(state="disabled") + self.player_status_var.set("Select a library to load tracks.") return num = count_audio_files(self.library_path) is_valid, _ = validate_soundvault_structure(self.library_path) @@ -2744,7 +2804,9 @@ def _validate_threshold(self) -> None: else: self.scan_btn.config(state="disabled") - def _play_preview(self, path: str) -> None: + def _play_preview( + self, path: str, start_ms: int = 30000, duration_ms: int = 15000 + ) -> None: """Play an audio preview, ensuring previous playback is cleaned up.""" # Stop any current preview immediately (no Tk calls inside) @@ -2757,9 +2819,16 @@ def _play_preview(self, path: str) -> None: ) return + if hasattr(self, "player_status_var"): + fname = os.path.basename(path) + suffix = " (30s highlight)" if duration_ms >= 30000 else "" + self.player_status_var.set(f"Playing {fname}{suffix}…") + def task() -> None: try: - self.preview_player.play_preview(path) + self.preview_player.play_preview( + path, start_ms=start_ms, duration_ms=duration_ms + ) except Exception as e: self.after(0, lambda: messagebox.showerror("Playback failed", str(e))) @@ -2767,8 +2836,8 @@ def task() -> None: self._preview_thread.start() def _preview_finished_ui(self): - # Placeholder for UI updates when preview completes - pass + if hasattr(self, "player_status_var"): + self.player_status_var.set("Ready to play another track.") def _load_thumbnail(self, path: str, size: int = 100) -> ImageTk.PhotoImage: img = None @@ -2868,6 +2937,120 @@ def do_cancel() -> None: top.wait_window() return result["ok"], skip_var.get() + # ── Player Tab Helpers ───────────────────────────────────────────── + def _clear_player_table(self) -> None: + if not hasattr(self, "player_tree"): + return + for row in self.player_tree.get_children(): + self.player_tree.delete(row) + self.player_tree_paths.clear() + + def _format_duration(self, seconds: float | None) -> str: + if not seconds: + return "—" + mins, secs = divmod(int(seconds), 60) + return f"{mins}:{secs:02d}" + + def _get_track_length(self, path: str) -> float | None: + try: + audio = MutagenFile(path) + if audio and getattr(audio, "info", None): + length = getattr(audio.info, "length", None) + if length: + return float(length) + except Exception: + return None + return None + + def _load_player_library_async(self) -> None: + if not self.library_path: + self.player_status_var.set("Select a library to load tracks.") + if hasattr(self, "player_reload_btn"): + self.player_reload_btn.config(state="disabled") + return + if self._player_load_thread and self._player_load_thread.is_alive(): + return + + self.player_status_var.set("Loading tracks…") + self.player_reload_btn.config(state="disabled") + thread = threading.Thread( + target=self._load_player_library, args=(self.library_path,), daemon=True + ) + self._player_load_thread = thread + thread.start() + + def _load_player_library(self, library_path: str) -> None: + try: + tracks = gather_tracks(library_path, self.folder_filter) + except Exception as exc: + self.after( + 0, + lambda: self.player_status_var.set( + f"Failed to load library: {exc}" + ), + ) + self.after(0, lambda: self.player_reload_btn.config(state="normal")) + return + + rows = [] + for path in tracks: + tags = get_tags(path) + title = tags.get("title") or os.path.basename(path) + artist = tags.get("artist") or "Unknown" + album = tags.get("album") or "" + length_sec = self._get_track_length(path) + rows.append( + { + "title": title, + "artist": artist, + "album": album, + "length": self._format_duration(length_sec), + "path": path, + } + ) + + self.after(0, lambda: self._update_player_table(rows, library_path)) + + def _update_player_table(self, rows: list[dict[str, str]], library_path: str) -> None: + if library_path != self.library_path: + return + self._clear_player_table() + self.player_tracks = rows + for row in rows: + item = self.player_tree.insert( + "", + "end", + values=( + row["title"], + row["artist"], + row["album"], + row["length"], + "▶", + ), + ) + self.player_tree_paths[item] = row["path"] + total = len(rows) + suffix = "track" if total == 1 else "tracks" + self.player_status_var.set( + f"Loaded {total} {suffix}. Click ▶ for a 30s highlight." + ) + self.player_reload_btn.config(state="normal") + + def _on_player_tree_click(self, event) -> None: + region = self.player_tree.identify_region(event.x, event.y) + if region != "cell": + return + col = self.player_tree.identify_column(event.x) + # Play column is the fifth heading (#5 when using ``show='headings'``) + if col != "#5": + return + item = self.player_tree.identify_row(event.y) + if not item: + return + path = self.player_tree_paths.get(item) + if path: + self._play_preview(path, duration_ms=30000) + def _send_help_query(self): threading.Thread(target=self._do_help_query, daemon=True).start() From 0f0f4464c1fb84466b7d36c5370b53ef27eabc1f Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sat, 13 Dec 2025 22:26:55 -0500 Subject: [PATCH 011/606] Open playlist output directory after creation --- main_gui.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/main_gui.py b/main_gui.py index 9d3335a..85691bf 100644 --- a/main_gui.py +++ b/main_gui.py @@ -304,6 +304,7 @@ def export_selected(): ) render_stats(result.get("genres")) append_log(f"Exported {len(selected)} playlist(s).") + open_genre_folder() def start_run(): nonlocal total_tracks @@ -456,6 +457,7 @@ def open_genre_folder(): progress_var = tk.StringVar(value="Waiting to start") running = tk.BooleanVar(value=False) total_tracks = 0 + playlists_dir: str | None = None q: queue.Queue = queue.Queue() cancel_event = threading.Event() @@ -534,7 +536,7 @@ def update_controls(): cancel_btn.config(state="normal" if running.get() else "disabled") def start_run(): - nonlocal total_tracks + nonlocal total_tracks, playlists_dir if running.get(): return path = app.require_library() @@ -548,6 +550,7 @@ def start_run(): messagebox.showinfo("No Tracks", "No audio files found in the library.") return total_tracks = len(tracks) + playlists_dir = os.path.join(path, "Playlists") cancel_event.clear() running.set(True) progress["value"] = 0 @@ -596,6 +599,12 @@ def poll_queue(): progress_var.set( f"{status} – processed {payload.get('processed')} of {payload.get('total')} tracks" ) + if ( + not payload.get("cancelled") + and playlists_dir + and payload.get("buckets") + ): + open_path(playlists_dir) update_controls() elif tag == "error": running.set(False) @@ -657,6 +666,19 @@ def poll_queue(): sel = tk.StringVar() count_var = tk.StringVar(value="20") + def open_path(path: str) -> None: + if not path: + return + try: + if sys.platform == "win32": + os.startfile(path) # type: ignore[attr-defined] + elif sys.platform == "darwin": + subprocess.run(["open", path], check=False) + else: + subprocess.run(["xdg-open", path], check=False) + except Exception as exc: # pragma: no cover - OS interaction + messagebox.showerror("Open File", f"Could not open {path}: {exc}") + def browse(): f = filedialog.askopenfilename() if f: @@ -678,6 +700,7 @@ def generate(): outfile = os.path.join(path, "Playlists", "autodj.m3u") write_playlist(order, outfile) messagebox.showinfo("Playlist", f"Written to {outfile}") + open_path(os.path.dirname(outfile)) row = ttk.Frame(frame) row.pack(padx=10, pady=10) From 6b9cc3c440db6fa2268c4c3a84aeaa03ec4b1ac5 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sat, 13 Dec 2025 22:34:42 -0500 Subject: [PATCH 012/606] Improve preview playback logging and ffplay isolation --- gui/audio_preview.py | 25 +++++- main_gui.py | 200 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 219 insertions(+), 6 deletions(-) diff --git a/gui/audio_preview.py b/gui/audio_preview.py index 45067b6..6ee4e51 100644 --- a/gui/audio_preview.py +++ b/gui/audio_preview.py @@ -1,3 +1,4 @@ +import logging import simpleaudio as sa import threading import subprocess @@ -15,9 +16,11 @@ def __init__(self, on_done=None) -> None: self._play_lock = threading.RLock() self._is_playing = False self._on_done = on_done + self._logger = logging.getLogger(__name__) def stop_preview(self) -> None: with self._play_lock: + self._logger.debug("Stopping preview playback") if self._play_obj is not None: try: self._play_obj.stop() @@ -49,6 +52,7 @@ def _monitor_playback(self, play_obj, proc) -> None: self._is_playing = False self._play_obj = None self._ffplay_proc = None + self._logger.debug("Preview playback finished") if self._on_done: try: self._on_done() @@ -67,6 +71,12 @@ def play_preview( from pydub import AudioSegment clip = AudioSegment.from_file(path)[start_ms : start_ms + duration_ms] + self._logger.debug( + "Playing preview via simpleaudio: path=%s start_ms=%s duration_ms=%s", + path, + start_ms, + duration_ms, + ) with self._play_lock: self._play_obj = sa.play_buffer( clip.raw_data, @@ -83,10 +93,16 @@ def play_preview( if not ffplay: with self._play_lock: self._is_playing = False + self._logger.exception( + "simpleaudio failed and ffplay not found: %s", sa_error + ) raise PlaybackError( f"simpleaudio failed and ffplay not found: {sa_error}" ) from sa_error try: + self._logger.warning( + "simpleaudio failed (%s); falling back to ffplay", sa_error + ) cmd = [ ffplay, "-nodisp", @@ -100,7 +116,13 @@ def play_preview( path, ] with self._play_lock: - self._ffplay_proc = subprocess.Popen(cmd, start_new_session=True) + self._logger.debug("Launching ffplay fallback: %s", " ".join(cmd)) + self._ffplay_proc = subprocess.Popen( + cmd, + start_new_session=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) proc = self._ffplay_proc threading.Thread( target=self._monitor_playback, args=(None, proc), daemon=True @@ -108,4 +130,5 @@ def play_preview( except Exception as exc: with self._play_lock: self._is_playing = False + self._logger.exception("ffplay failed: %s", exc) raise PlaybackError(f"ffplay failed: {exc}") from exc diff --git a/main_gui.py b/main_gui.py index 9d3335a..5a8815e 100644 --- a/main_gui.py +++ b/main_gui.py @@ -971,6 +971,12 @@ def __init__(self): on_done=lambda: self.after(0, self._preview_finished_ui) ) + # Player tab state + self.player_tracks: list[dict[str, str]] = [] + self.player_tree_paths: dict[str, str] = {} + self.player_status_var = tk.StringVar(value="Select a library to load tracks.") + self._player_load_thread: threading.Thread | None = None + # assume ffmpeg is available without performing checks self.ffmpeg_available = True @@ -1367,6 +1373,54 @@ def build_ui(self): self.scan_btn.config(state="normal") self._validate_threshold() + # ─── Player Tab ──────────────────────────────────────────────── + self.player_tab = ttk.Frame(self.notebook) + self.notebook.add(self.player_tab, text="Player") + + player_controls = ttk.Frame(self.player_tab) + player_controls.pack(fill="x", padx=10, pady=(10, 5)) + ttk.Label(player_controls, textvariable=self.player_status_var).pack( + side="left" + ) + self.player_reload_btn = ttk.Button( + player_controls, + text="Reload", + command=self._load_player_library_async, + state="disabled", + ) + self.player_reload_btn.pack(side="right") + + player_table = ttk.Frame(self.player_tab) + player_table.pack(fill="both", expand=True, padx=10, pady=(0, 10)) + p_vsb = ttk.Scrollbar(player_table, orient="vertical") + p_vsb.pack(side="right", fill="y") + p_hsb = ttk.Scrollbar(player_table, orient="horizontal") + p_hsb.pack(side="bottom", fill="x") + + cols = ("Title", "Artist", "Album", "Length", "Play") + self.player_tree = ttk.Treeview( + player_table, + columns=cols, + show="headings", + yscrollcommand=p_vsb.set, + xscrollcommand=p_hsb.set, + ) + p_vsb.config(command=self.player_tree.yview) + p_hsb.config(command=self.player_tree.xview) + self.player_tree.pack(fill="both", expand=True) + + widths = {"Title": 220, "Artist": 140, "Album": 160, "Length": 70, "Play": 50} + for c in cols: + self.player_tree.heading(c, text=c) + self.player_tree.column( + c, + width=widths.get(c, 100), + anchor="center" if c in {"Length", "Play"} else "w", + stretch=c != "Play", + ) + + self.player_tree.bind("", self._on_player_tree_click) + # ─── Library Sync Tab ───────────────────────────────────────────── self.sync_tab = ttk.Frame(self.notebook) self.notebook.add(self.sync_tab, text="Library Sync") @@ -1510,10 +1564,16 @@ def select_library(self): self.scan_btn.config(state="normal") self._validate_threshold() self.update_library_info() + if hasattr(self, "player_reload_btn"): + self.player_reload_btn.config(state="normal") + self._load_player_library_async() def update_library_info(self): if not self.library_path: self.library_stats_var.set("") + if hasattr(self, "player_reload_btn"): + self.player_reload_btn.config(state="disabled") + self.player_status_var.set("Select a library to load tracks.") return num = count_audio_files(self.library_path) is_valid, _ = validate_soundvault_structure(self.library_path) @@ -2744,7 +2804,9 @@ def _validate_threshold(self) -> None: else: self.scan_btn.config(state="disabled") - def _play_preview(self, path: str) -> None: + def _play_preview( + self, path: str, start_ms: int = 30000, duration_ms: int = 15000 + ) -> None: """Play an audio preview, ensuring previous playback is cleaned up.""" # Stop any current preview immediately (no Tk calls inside) @@ -2757,18 +2819,32 @@ def _play_preview(self, path: str) -> None: ) return + if hasattr(self, "player_status_var"): + fname = os.path.basename(path) + suffix = " (30s highlight)" if duration_ms >= 30000 else "" + self.player_status_var.set(f"Playing {fname}{suffix}…") + def task() -> None: try: - self.preview_player.play_preview(path) + self.preview_player.play_preview( + path, start_ms=start_ms, duration_ms=duration_ms + ) except Exception as e: - self.after(0, lambda: messagebox.showerror("Playback failed", str(e))) + logging.exception("Preview playback failed") + self.after( + 0, + lambda: ( + self.player_status_var.set("Playback failed. Check logs."), + messagebox.showerror("Playback failed", str(e)), + ), + ) self._preview_thread = threading.Thread(target=task, daemon=True) self._preview_thread.start() def _preview_finished_ui(self): - # Placeholder for UI updates when preview completes - pass + if hasattr(self, "player_status_var"): + self.player_status_var.set("Ready to play another track.") def _load_thumbnail(self, path: str, size: int = 100) -> ImageTk.PhotoImage: img = None @@ -2868,6 +2944,120 @@ def do_cancel() -> None: top.wait_window() return result["ok"], skip_var.get() + # ── Player Tab Helpers ───────────────────────────────────────────── + def _clear_player_table(self) -> None: + if not hasattr(self, "player_tree"): + return + for row in self.player_tree.get_children(): + self.player_tree.delete(row) + self.player_tree_paths.clear() + + def _format_duration(self, seconds: float | None) -> str: + if not seconds: + return "—" + mins, secs = divmod(int(seconds), 60) + return f"{mins}:{secs:02d}" + + def _get_track_length(self, path: str) -> float | None: + try: + audio = MutagenFile(path) + if audio and getattr(audio, "info", None): + length = getattr(audio.info, "length", None) + if length: + return float(length) + except Exception: + return None + return None + + def _load_player_library_async(self) -> None: + if not self.library_path: + self.player_status_var.set("Select a library to load tracks.") + if hasattr(self, "player_reload_btn"): + self.player_reload_btn.config(state="disabled") + return + if self._player_load_thread and self._player_load_thread.is_alive(): + return + + self.player_status_var.set("Loading tracks…") + self.player_reload_btn.config(state="disabled") + thread = threading.Thread( + target=self._load_player_library, args=(self.library_path,), daemon=True + ) + self._player_load_thread = thread + thread.start() + + def _load_player_library(self, library_path: str) -> None: + try: + tracks = gather_tracks(library_path, self.folder_filter) + except Exception as exc: + self.after( + 0, + lambda: self.player_status_var.set( + f"Failed to load library: {exc}" + ), + ) + self.after(0, lambda: self.player_reload_btn.config(state="normal")) + return + + rows = [] + for path in tracks: + tags = get_tags(path) + title = tags.get("title") or os.path.basename(path) + artist = tags.get("artist") or "Unknown" + album = tags.get("album") or "" + length_sec = self._get_track_length(path) + rows.append( + { + "title": title, + "artist": artist, + "album": album, + "length": self._format_duration(length_sec), + "path": path, + } + ) + + self.after(0, lambda: self._update_player_table(rows, library_path)) + + def _update_player_table(self, rows: list[dict[str, str]], library_path: str) -> None: + if library_path != self.library_path: + return + self._clear_player_table() + self.player_tracks = rows + for row in rows: + item = self.player_tree.insert( + "", + "end", + values=( + row["title"], + row["artist"], + row["album"], + row["length"], + "▶", + ), + ) + self.player_tree_paths[item] = row["path"] + total = len(rows) + suffix = "track" if total == 1 else "tracks" + self.player_status_var.set( + f"Loaded {total} {suffix}. Click ▶ for a 30s highlight." + ) + self.player_reload_btn.config(state="normal") + + def _on_player_tree_click(self, event) -> None: + region = self.player_tree.identify_region(event.x, event.y) + if region != "cell": + return + col = self.player_tree.identify_column(event.x) + # Play column is the fifth heading (#5 when using ``show='headings'``) + if col != "#5": + return + item = self.player_tree.identify_row(event.y) + if not item: + return + path = self.player_tree_paths.get(item) + if path: + self._play_preview(path, duration_ms=30000) + def _send_help_query(self): threading.Thread(target=self._do_help_query, daemon=True).start() From 588d4c4f370952697a00b5bbd78d96cabfbe15ae Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sat, 13 Dec 2025 22:36:01 -0500 Subject: [PATCH 013/606] Remove placeholder Metadata playlist tool --- main_gui.py | 1 - 1 file changed, 1 deletion(-) diff --git a/main_gui.py b/main_gui.py index dfc847e..fce2c24 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1219,7 +1219,6 @@ def build_ui(self): "Interactive – HDBSCAN", "Sort by Genre", "Tempo/Energy Buckets", - "Metadata", "Auto-DJ", ]: self.plugin_list.insert("end", name) From 56baa278a2c5042cf624b362f0351b57a1228d48 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sat, 13 Dec 2025 22:43:46 -0500 Subject: [PATCH 014/606] Add VLC-based preview backend with UI gating --- gui/audio_preview.py | 200 ++++++++++++++++++++----------------------- main_gui.py | 141 +++++++++++++++++++++++------- 2 files changed, 205 insertions(+), 136 deletions(-) diff --git a/gui/audio_preview.py b/gui/audio_preview.py index 6ee4e51..4ffd7b4 100644 --- a/gui/audio_preview.py +++ b/gui/audio_preview.py @@ -1,134 +1,122 @@ import logging -import simpleaudio as sa import threading -import subprocess -import shutil +from typing import Optional + + +try: + import vlc + + VLC_AVAILABLE = True + VLC_IMPORT_ERROR: Exception | None = None +except Exception as exc: # pragma: no cover - import guard + VLC_AVAILABLE = False + VLC_IMPORT_ERROR = exc class PlaybackError(Exception): pass -class PreviewPlayer: +class VlcPreviewPlayer: def __init__(self, on_done=None) -> None: - self._play_obj = None - self._ffplay_proc = None + self._logger = logging.getLogger(__name__) + self._on_done = on_done self._play_lock = threading.RLock() + self._instance: Optional["vlc.Instance"] = None + self._player: Optional["vlc.MediaPlayer"] = None + self._event_mgr = None self._is_playing = False - self._on_done = on_done - self._logger = logging.getLogger(__name__) + self._init_error: Exception | None = VLC_IMPORT_ERROR + self._setup_player() + + def _setup_player(self) -> None: + if self._init_error: + self._logger.error("python-vlc not available: %s", self._init_error) + return + + try: + self._instance = vlc.Instance() + self._player = self._instance.media_player_new() + self._event_mgr = self._player.event_manager() + self._event_mgr.event_attach( + vlc.EventType.MediaPlayerEndReached, self._handle_end + ) + self._event_mgr.event_attach( + vlc.EventType.MediaPlayerStopped, self._handle_end + ) + self._event_mgr.event_attach( + vlc.EventType.MediaPlayerEncounteredError, self._handle_error + ) + self._logger.debug("VLC preview backend initialized") + except Exception as exc: + self._init_error = exc + self._logger.exception("Failed to initialize VLC preview backend") + + @property + def available(self) -> bool: + return self._init_error is None + + @property + def availability_error(self) -> str | None: + if self._init_error: + return str(self._init_error) + return None def stop_preview(self) -> None: with self._play_lock: - self._logger.debug("Stopping preview playback") - if self._play_obj is not None: - try: - self._play_obj.stop() - except Exception: - pass - self._play_obj = None - if self._ffplay_proc is not None: - try: - self._ffplay_proc.terminate() - except Exception: - pass - self._ffplay_proc = None + if not self._player: + return + try: + self._logger.debug("Stopping VLC preview playback") + self._player.stop() + except Exception: + self._logger.exception("Error while stopping VLC preview") self._is_playing = False - def _monitor_playback(self, play_obj, proc) -> None: - try: - if play_obj is not None: - try: - play_obj.wait_done() - except Exception: - pass - elif proc is not None: - try: - proc.wait() - except Exception: - pass - finally: - with self._play_lock: - self._is_playing = False - self._play_obj = None - self._ffplay_proc = None - self._logger.debug("Preview playback finished") - if self._on_done: - try: - self._on_done() - except Exception: - pass - - def play_preview( - self, path: str, start_ms: int = 30000, duration_ms: int = 15000 - ) -> None: + def play_clip(self, path: str, start_ms: int = 30000, duration_ms: int = 15000) -> None: with self._play_lock: + if not self.available or not self._player or not self._instance: + raise PlaybackError(self.availability_error or "VLC backend unavailable") if self._is_playing: + self._logger.debug("Stopping existing preview before starting new one") self.stop_preview() self._is_playing = True - try: - from pydub import AudioSegment + media = self._instance.media_new(path) + media.add_option(":no-video") + media.add_option(f":start-time={start_ms / 1000}") + media.add_option(f":run-time={duration_ms / 1000}") - clip = AudioSegment.from_file(path)[start_ms : start_ms + duration_ms] - self._logger.debug( - "Playing preview via simpleaudio: path=%s start_ms=%s duration_ms=%s", + self._logger.info( + "Starting VLC preview: path=%s start_ms=%s duration_ms=%s", path, start_ms, duration_ms, ) - with self._play_lock: - self._play_obj = sa.play_buffer( - clip.raw_data, - num_channels=clip.channels, - bytes_per_sample=clip.sample_width, - sample_rate=clip.frame_rate, - ) - play_obj = self._play_obj - threading.Thread( - target=self._monitor_playback, args=(play_obj, None), daemon=True - ).start() - except Exception as sa_error: - ffplay = shutil.which("ffplay") - if not ffplay: - with self._play_lock: - self._is_playing = False - self._logger.exception( - "simpleaudio failed and ffplay not found: %s", sa_error - ) - raise PlaybackError( - f"simpleaudio failed and ffplay not found: {sa_error}" - ) from sa_error + self._player.set_media(media) try: - self._logger.warning( - "simpleaudio failed (%s); falling back to ffplay", sa_error - ) - cmd = [ - ffplay, - "-nodisp", - "-autoexit", - "-loglevel", - "quiet", - "-ss", - str(start_ms / 1000), - "-t", - str(duration_ms / 1000), - path, - ] - with self._play_lock: - self._logger.debug("Launching ffplay fallback: %s", " ".join(cmd)) - self._ffplay_proc = subprocess.Popen( - cmd, - start_new_session=True, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - proc = self._ffplay_proc - threading.Thread( - target=self._monitor_playback, args=(None, proc), daemon=True - ).start() + self._player.play() except Exception as exc: - with self._play_lock: - self._is_playing = False - self._logger.exception("ffplay failed: %s", exc) - raise PlaybackError(f"ffplay failed: {exc}") from exc + self._is_playing = False + self._logger.exception("Failed to start VLC playback") + raise PlaybackError(str(exc)) from exc + + def _handle_end(self, event=None) -> None: # pragma: no cover - event driven + with self._play_lock: + self._is_playing = False + self._logger.debug("VLC preview finished") + if self._on_done: + try: + self._on_done() + except Exception: + self._logger.exception("Error in preview completion callback") + + def _handle_error(self, event=None) -> None: # pragma: no cover - event driven + with self._play_lock: + self._is_playing = False + self._logger.error("VLC reported an error during preview playback") + if self._on_done: + try: + self._on_done() + except Exception: + self._logger.exception("Error in preview error callback") diff --git a/main_gui.py b/main_gui.py index 65f70d0..534f0c9 100644 --- a/main_gui.py +++ b/main_gui.py @@ -44,9 +44,8 @@ import chromaprint_utils from controllers.library_index_controller import generate_index from controllers.genre_list_controller import list_unique_genres -from controllers.highlight_controller import PYDUB_AVAILABLE from controllers.scan_progress_controller import ScanProgressController -from gui.audio_preview import PreviewPlayer +from gui.audio_preview import PlaybackError, VlcPreviewPlayer from io import BytesIO from PIL import Image, ImageTk from mutagen import File as MutagenFile @@ -990,9 +989,20 @@ def __init__(self): # Quality Checker state self._dup_logging = False self._preview_thread = None - self.preview_player = PreviewPlayer( + self.preview_player = VlcPreviewPlayer( on_done=lambda: self.after(0, self._preview_finished_ui) ) + self.preview_backend_available = self.preview_player.available + self.preview_backend_error = self.preview_player.availability_error + if not self.preview_backend_available: + reason = self.preview_backend_error or "Install python-vlc to enable playback." + logging.error("Preview backend unavailable: %s", reason) + self.player_status_var.set(f"Preview disabled: {reason}") + else: + logging.info("VLC preview backend initialized") + self._quality_play_buttons: list[ttk.Button] = [] + self._preview_in_progress = False + self._player_busy_item: str | None = None # Player tab state self.player_tracks: list[dict[str, str]] = [] @@ -2827,18 +2837,25 @@ def _validate_threshold(self) -> None: self.scan_btn.config(state="disabled") def _play_preview( - self, path: str, start_ms: int = 30000, duration_ms: int = 15000 + self, + path: str, + start_ms: int = 30000, + duration_ms: int = 15000, + player_item: str | None = None, ) -> None: - """Play an audio preview, ensuring previous playback is cleaned up.""" - - # Stop any current preview immediately (no Tk calls inside) - self.preview_player.stop_preview() + """Play an audio preview while serializing concurrent requests.""" + + if not self.preview_backend_available: + err = self.preview_backend_error or "VLC preview backend unavailable." + logging.error("Preview backend unavailable: %s", err) + if hasattr(self, "player_status_var"): + self.player_status_var.set(f"Preview disabled: {err}") + messagebox.showerror("Playback failed", err) + return - if not PYDUB_AVAILABLE: - messagebox.showerror( - "Playback failed", - "pydub/ffmpeg not available. Install requirements to enable preview.", - ) + if self._preview_in_progress: + if hasattr(self, "player_status_var"): + self.player_status_var.set("Finishing current preview…") return if hasattr(self, "player_status_var"): @@ -2846,27 +2863,65 @@ def _play_preview( suffix = " (30s highlight)" if duration_ms >= 30000 else "" self.player_status_var.set(f"Playing {fname}{suffix}…") + self._preview_in_progress = True + self._set_preview_controls_state(enabled=False) + self._set_player_play_state_busy(player_item) + def task() -> None: try: - self.preview_player.play_preview( + self.preview_player.play_clip( path, start_ms=start_ms, duration_ms=duration_ms ) - except Exception as e: + except PlaybackError as e: logging.exception("Preview playback failed") - self.after( - 0, - lambda: ( - self.player_status_var.set("Playback failed. Check logs."), - messagebox.showerror("Playback failed", str(e)), - ), - ) + self.after(0, lambda: self._preview_finished_ui(error=str(e))) + except Exception as e: # pragma: no cover - safety net + logging.exception("Unexpected preview failure") + self.after(0, lambda: self._preview_finished_ui(error=str(e))) self._preview_thread = threading.Thread(target=task, daemon=True) self._preview_thread.start() - def _preview_finished_ui(self): + def _preview_finished_ui(self, error: str | None = None): + self._preview_in_progress = False + self._restore_player_play_icons() + self._set_preview_controls_state(enabled=self.preview_backend_available) if hasattr(self, "player_status_var"): - self.player_status_var.set("Ready to play another track.") + if error: + self.player_status_var.set("Playback failed. Check logs.") + messagebox.showerror("Playback failed", error) + else: + self.player_status_var.set("Ready to play another track.") + + def _set_preview_controls_state(self, enabled: bool) -> None: + state = "!disabled" if enabled else "disabled" + for btn in getattr(self, "_quality_play_buttons", []): + try: + btn.state([state]) + except Exception: + logging.exception("Failed to update preview button state") + + def _set_player_play_state_busy(self, item_id: str | None) -> None: + if not hasattr(self, "player_tree"): + return + self._restore_player_play_icons() + if not item_id or not self.player_tree.exists(item_id): + return + values = list(self.player_tree.item(item_id, "values")) + if len(values) >= 5: + values[4] = "⏳" + self.player_tree.item(item_id, values=values) + self._player_busy_item = item_id + + def _restore_player_play_icons(self) -> None: + if not hasattr(self, "player_tree"): + return + if self._player_busy_item and self.player_tree.exists(self._player_busy_item): + values = list(self.player_tree.item(self._player_busy_item, "values")) + if len(values) >= 5: + values[4] = "▶" if self.preview_backend_available else "—" + self.player_tree.item(self._player_busy_item, values=values) + self._player_busy_item = None def _load_thumbnail(self, path: str, size: int = 100) -> ImageTk.PhotoImage: img = None @@ -2893,6 +2948,7 @@ def _load_thumbnail(self, path: str, size: int = 100) -> ImageTk.PhotoImage: def populate_quality_table(self, matches): self.clear_quality_view() + self._quality_play_buttons = [] if not matches: ttk.Label(self.qc_inner, text="No duplicates detected in this library").pack(pady=20) return @@ -2901,8 +2957,18 @@ def populate_quality_table(self, matches): row = ttk.Frame(self.qc_inner) row.pack(fill="x", padx=10, pady=2) - play_btn = ttk.Button(row, text="▶", width=1, command=lambda p=dup_path: self._play_preview(p)) + btn_state = "normal" + if not self.preview_backend_available or self._preview_in_progress: + btn_state = "disabled" + play_btn = ttk.Button( + row, + text="▶", + width=1, + state=btn_state, + command=lambda p=dup_path: self._play_preview(p), + ) play_btn.pack(side="left") + self._quality_play_buttons.append(play_btn) thumb = self._load_thumbnail(dup_path, size=40) img_label = ttk.Label(row, image=thumb) @@ -3045,6 +3111,7 @@ def _update_player_table(self, rows: list[dict[str, str]], library_path: str) -> return self._clear_player_table() self.player_tracks = rows + play_icon = "▶" if self.preview_backend_available else "—" for row in rows: item = self.player_tree.insert( "", @@ -3054,15 +3121,21 @@ def _update_player_table(self, rows: list[dict[str, str]], library_path: str) -> row["artist"], row["album"], row["length"], - "▶", + play_icon, ), ) self.player_tree_paths[item] = row["path"] total = len(rows) suffix = "track" if total == 1 else "tracks" - self.player_status_var.set( - f"Loaded {total} {suffix}. Click ▶ for a 30s highlight." - ) + if not self.preview_backend_available: + reason = self.preview_backend_error or "Install python-vlc to enable playback." + self.player_status_var.set( + f"Loaded {total} {suffix}. Preview disabled: {reason}" + ) + else: + self.player_status_var.set( + f"Loaded {total} {suffix}. Click ▶ for a 30s highlight." + ) self.player_reload_btn.config(state="normal") def _on_player_tree_click(self, event) -> None: @@ -3076,9 +3149,17 @@ def _on_player_tree_click(self, event) -> None: item = self.player_tree.identify_row(event.y) if not item: return + if not self.preview_backend_available: + self.player_status_var.set( + f"Preview disabled: {self.preview_backend_error or 'Install python-vlc.'}" + ) + return + if self._preview_in_progress: + self.player_status_var.set("Finishing current preview…") + return path = self.player_tree_paths.get(item) if path: - self._play_preview(path, duration_ms=30000) + self._play_preview(path, duration_ms=30000, player_item=item) def _send_help_query(self): threading.Thread(target=self._do_help_query, daemon=True).start() From aaf8c1a851cb9a18834738b37f33433fdc9abadd Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 14 Dec 2025 07:25:04 -0500 Subject: [PATCH 015/606] Fix player status initialization before preview checks --- main_gui.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/main_gui.py b/main_gui.py index 534f0c9..e328abc 100644 --- a/main_gui.py +++ b/main_gui.py @@ -989,6 +989,9 @@ def __init__(self): # Quality Checker state self._dup_logging = False self._preview_thread = None + self.player_status_var = tk.StringVar( + value="Select a library to load tracks." + ) self.preview_player = VlcPreviewPlayer( on_done=lambda: self.after(0, self._preview_finished_ui) ) @@ -1007,7 +1010,6 @@ def __init__(self): # Player tab state self.player_tracks: list[dict[str, str]] = [] self.player_tree_paths: dict[str, str] = {} - self.player_status_var = tk.StringVar(value="Select a library to load tracks.") self._player_load_thread: threading.Thread | None = None # assume ffmpeg is available without performing checks From a81aa8421f0ccaf9265286cf7cbb0af6f6bf4948 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 14 Dec 2025 07:34:24 -0500 Subject: [PATCH 016/606] Add missing runtime dependencies to requirements --- main_gui.py | 4 +++- requirements.txt | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/main_gui.py b/main_gui.py index 534f0c9..e328abc 100644 --- a/main_gui.py +++ b/main_gui.py @@ -989,6 +989,9 @@ def __init__(self): # Quality Checker state self._dup_logging = False self._preview_thread = None + self.player_status_var = tk.StringVar( + value="Select a library to load tracks." + ) self.preview_player = VlcPreviewPlayer( on_done=lambda: self.after(0, self._preview_finished_ui) ) @@ -1007,7 +1010,6 @@ def __init__(self): # Player tab state self.player_tracks: list[dict[str, str]] = [] self.player_tree_paths: dict[str, str] = {} - self.player_status_var = tk.StringVar(value="Select a library to load tracks.") self._player_load_thread: threading.Thread | None = None # assume ffmpeg is available without performing checks diff --git a/requirements.txt b/requirements.txt index 9e35f2d..180b732 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ simpleaudio ffmpeg-python pyacoustid musicbrainzngs +python-vlc # essentia # disabled on Windows librosa scipy @@ -11,7 +12,9 @@ numpy scikit-learn hdbscan matplotlib +Pillow requests +typing_extensions # optional metadata helpers spotipy tidal-dl From 4ea3f35abd1a4393d17a57d68b16216cb4538cc2 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 14 Dec 2025 07:37:12 -0500 Subject: [PATCH 017/606] Allow interrupting audio previews --- main_gui.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/main_gui.py b/main_gui.py index e328abc..5184cc2 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1006,6 +1006,7 @@ def __init__(self): self._quality_play_buttons: list[ttk.Button] = [] self._preview_in_progress = False self._player_busy_item: str | None = None + self._ignore_next_preview_finish = False # Player tab state self.player_tracks: list[dict[str, str]] = [] @@ -2856,9 +2857,10 @@ def _play_preview( return if self._preview_in_progress: - if hasattr(self, "player_status_var"): - self.player_status_var.set("Finishing current preview…") - return + # Interrupt any existing playback before starting the new preview + self._ignore_next_preview_finish = True + self.preview_player.stop_preview() + self._preview_in_progress = False if hasattr(self, "player_status_var"): fname = os.path.basename(path) @@ -2866,7 +2868,6 @@ def _play_preview( self.player_status_var.set(f"Playing {fname}{suffix}…") self._preview_in_progress = True - self._set_preview_controls_state(enabled=False) self._set_player_play_state_busy(player_item) def task() -> None: @@ -2885,6 +2886,11 @@ def task() -> None: self._preview_thread.start() def _preview_finished_ui(self, error: str | None = None): + if self._ignore_next_preview_finish: + # Skip UI reset triggered by an intentionally interrupted preview + self._ignore_next_preview_finish = False + return + self._preview_in_progress = False self._restore_player_play_icons() self._set_preview_controls_state(enabled=self.preview_backend_available) From a978631a6369ace8a82e9f3247561e465b0842c0 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 14 Dec 2025 07:45:35 -0500 Subject: [PATCH 018/606] Revert "Allow interrupting audio previews" --- main_gui.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/main_gui.py b/main_gui.py index 5184cc2..e328abc 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1006,7 +1006,6 @@ def __init__(self): self._quality_play_buttons: list[ttk.Button] = [] self._preview_in_progress = False self._player_busy_item: str | None = None - self._ignore_next_preview_finish = False # Player tab state self.player_tracks: list[dict[str, str]] = [] @@ -2857,10 +2856,9 @@ def _play_preview( return if self._preview_in_progress: - # Interrupt any existing playback before starting the new preview - self._ignore_next_preview_finish = True - self.preview_player.stop_preview() - self._preview_in_progress = False + if hasattr(self, "player_status_var"): + self.player_status_var.set("Finishing current preview…") + return if hasattr(self, "player_status_var"): fname = os.path.basename(path) @@ -2868,6 +2866,7 @@ def _play_preview( self.player_status_var.set(f"Playing {fname}{suffix}…") self._preview_in_progress = True + self._set_preview_controls_state(enabled=False) self._set_player_play_state_busy(player_item) def task() -> None: @@ -2886,11 +2885,6 @@ def task() -> None: self._preview_thread.start() def _preview_finished_ui(self, error: str | None = None): - if self._ignore_next_preview_finish: - # Skip UI reset triggered by an intentionally interrupted preview - self._ignore_next_preview_finish = False - return - self._preview_in_progress = False self._restore_player_play_icons() self._set_preview_controls_state(enabled=self.preview_backend_available) From 688be3d481aa85d370dcb12d007ac3758e4a85bc Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 14 Dec 2025 07:51:18 -0500 Subject: [PATCH 019/606] Allow preview clicks to interrupt playback --- main_gui.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/main_gui.py b/main_gui.py index e328abc..4152fb8 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1006,6 +1006,7 @@ def __init__(self): self._quality_play_buttons: list[ttk.Button] = [] self._preview_in_progress = False self._player_busy_item: str | None = None + self._ignore_next_preview_finish = False # Player tab state self.player_tracks: list[dict[str, str]] = [] @@ -2856,9 +2857,10 @@ def _play_preview( return if self._preview_in_progress: - if hasattr(self, "player_status_var"): - self.player_status_var.set("Finishing current preview…") - return + # Interrupt any existing playback before starting the new preview + self._ignore_next_preview_finish = True + self.preview_player.stop_preview() + self._preview_in_progress = False if hasattr(self, "player_status_var"): fname = os.path.basename(path) @@ -2866,7 +2868,6 @@ def _play_preview( self.player_status_var.set(f"Playing {fname}{suffix}…") self._preview_in_progress = True - self._set_preview_controls_state(enabled=False) self._set_player_play_state_busy(player_item) def task() -> None: @@ -2885,6 +2886,11 @@ def task() -> None: self._preview_thread.start() def _preview_finished_ui(self, error: str | None = None): + if self._ignore_next_preview_finish: + # Skip UI reset triggered by an intentionally interrupted preview + self._ignore_next_preview_finish = False + return + self._preview_in_progress = False self._restore_player_play_icons() self._set_preview_controls_state(enabled=self.preview_backend_available) @@ -3156,9 +3162,6 @@ def _on_player_tree_click(self, event) -> None: f"Preview disabled: {self.preview_backend_error or 'Install python-vlc.'}" ) return - if self._preview_in_progress: - self.player_status_var.set("Finishing current preview…") - return path = self.player_tree_paths.get(item) if path: self._play_preview(path, duration_ms=30000, player_item=item) From fc0400157c6ec89cb709a60039cf91f965389574 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 14 Dec 2025 07:54:44 -0500 Subject: [PATCH 020/606] Silence VLC logging for previews --- gui/audio_preview.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gui/audio_preview.py b/gui/audio_preview.py index 4ffd7b4..c81d97f 100644 --- a/gui/audio_preview.py +++ b/gui/audio_preview.py @@ -1,8 +1,11 @@ import logging +import os import threading from typing import Optional +os.environ["VLC_VERBOSE"] = "0" + try: import vlc @@ -35,7 +38,7 @@ def _setup_player(self) -> None: return try: - self._instance = vlc.Instance() + self._instance = vlc.Instance("--quiet", "--no-video-title-show", "--no-stats") self._player = self._instance.media_player_new() self._event_mgr = self._player.event_manager() self._event_mgr.event_attach( From 1c89ebd55295388b5dce5608a41e9f1ee0b1915a Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 14 Dec 2025 07:59:33 -0500 Subject: [PATCH 021/606] Show library stats on one line --- main_gui.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main_gui.py b/main_gui.py index 4152fb8..cdfa00e 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1613,7 +1613,7 @@ def update_library_info(self): num = count_audio_files(self.library_path) is_valid, _ = validate_soundvault_structure(self.library_path) status = "Valid" if is_valid else "Invalid" - self.library_stats_var.set(f"Songs: {num}\nValidation: {status}") + self.library_stats_var.set(f"Songs: {num} Validation: {status}") def require_library(self): if not self.library_path: From a4937c19e5d2c3ebe27577f2e0718446ca8853f3 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 14 Dec 2025 08:08:14 -0500 Subject: [PATCH 022/606] Add album art preview to player tab --- main_gui.py | 118 ++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 100 insertions(+), 18 deletions(-) diff --git a/main_gui.py b/main_gui.py index cdfa00e..29c67ae 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1011,7 +1011,9 @@ def __init__(self): # Player tab state self.player_tracks: list[dict[str, str]] = [] self.player_tree_paths: dict[str, str] = {} + self.player_tree_rows: dict[str, dict[str, str]] = {} self._player_load_thread: threading.Thread | None = None + self.player_art_image: ImageTk.PhotoImage | None = None # assume ffmpeg is available without performing checks self.ffmpeg_available = True @@ -1425,8 +1427,11 @@ def build_ui(self): ) self.player_reload_btn.pack(side="right") - player_table = ttk.Frame(self.player_tab) - player_table.pack(fill="both", expand=True, padx=10, pady=(0, 10)) + player_content = ttk.Frame(self.player_tab) + player_content.pack(fill="both", expand=True, padx=10, pady=(0, 10)) + + player_table = ttk.Frame(player_content) + player_table.pack(side="left", fill="both", expand=True) p_vsb = ttk.Scrollbar(player_table, orient="vertical") p_vsb.pack(side="right", fill="y") p_hsb = ttk.Scrollbar(player_table, orient="horizontal") @@ -1455,6 +1460,20 @@ def build_ui(self): ) self.player_tree.bind("", self._on_player_tree_click) + self.player_tree.bind("<>", self._on_player_selection_change) + + art_panel = ttk.Frame(player_content) + art_panel.pack(side="right", fill="y", padx=(10, 0)) + self.player_art_label = ttk.Label(art_panel) + self.player_art_label.pack(fill="x", pady=(0, 6)) + self.player_art_caption = ttk.Label( + art_panel, + text="Select a track to view album art", + wraplength=220, + justify="center", + ) + self.player_art_caption.pack(fill="x") + self._update_player_art(None) # ─── Library Sync Tab ───────────────────────────────────────────── self.sync_tab = ttk.Frame(self.notebook) @@ -2931,29 +2950,66 @@ def _restore_player_play_icons(self) -> None: self.player_tree.item(self._player_busy_item, values=values) self._player_busy_item = None - def _load_thumbnail(self, path: str, size: int = 100) -> ImageTk.PhotoImage: + def _player_art_size(self) -> int: + base = 200 + scale_factor = 0.8 + (0.2 * max(self.current_scale, 1.0)) + return int(base * min(scale_factor, 1.25)) + + def _load_thumbnail(self, path: str | None, size: int = 100) -> ImageTk.PhotoImage: img = None - try: - audio = MutagenFile(path) - img_data = None - if hasattr(audio, "tags") and audio.tags is not None: - for key in audio.tags.keys(): - if str(key).startswith("APIC"): - img_data = audio.tags[key].data + if path: + try: + audio = MutagenFile(path) + img_data = None + if hasattr(audio, "tags") and audio.tags is not None: + for key in audio.tags.keys(): + if str(key).startswith("APIC"): + img_data = audio.tags[key].data + break + if img_data is None and getattr(audio, "pictures", None): + pics = getattr(audio, "pictures", []) + if pics: + img_data = pics[0].data + if img_data: + img = Image.open(BytesIO(img_data)) + except Exception: + img = None + if img is None: + candidates = [] + folder = os.path.dirname(path) + for stem in ("cover", "folder", "front", "album"): + for ext in ("jpg", "jpeg", "png", "webp"): + candidates.append(os.path.join(folder, f"{stem}.{ext}")) + for candidate in candidates: + if not os.path.exists(candidate): + continue + try: + img = Image.open(candidate) break - if img_data is None and getattr(audio, "pictures", None): - pics = getattr(audio, "pictures", []) - if pics: - img_data = pics[0].data - if img_data: - img = Image.open(BytesIO(img_data)) - except Exception: - img = None + except Exception: + img = None if img is None: img = Image.new("RGB", (size, size), "#777777") img.thumbnail((size, size)) return ImageTk.PhotoImage(img) + def _update_player_art( + self, path: str | None, title: str | None = None, artist: str | None = None + ) -> None: + if not hasattr(self, "player_art_label"): + return + size = self._player_art_size() + art = self._load_thumbnail(path, size=size) + self.player_art_image = art + self.player_art_label.configure(image=art) + if path: + caption = title or os.path.basename(path) + if artist: + caption = f"{caption}\n{artist}" + else: + caption = "Select a track to view album art" + self.player_art_caption.configure(text=caption, wraplength=size + 20) + def populate_quality_table(self, matches): self.clear_quality_view() self._quality_play_buttons = [] @@ -3047,6 +3103,8 @@ def _clear_player_table(self) -> None: for row in self.player_tree.get_children(): self.player_tree.delete(row) self.player_tree_paths.clear() + self.player_tree_rows.clear() + self._update_player_art(None) def _format_duration(self, seconds: float | None) -> str: if not seconds: @@ -3133,6 +3191,7 @@ def _update_player_table(self, rows: list[dict[str, str]], library_path: str) -> ), ) self.player_tree_paths[item] = row["path"] + self.player_tree_rows[item] = row total = len(rows) suffix = "track" if total == 1 else "tracks" if not self.preview_backend_available: @@ -3145,6 +3204,13 @@ def _update_player_table(self, rows: list[dict[str, str]], library_path: str) -> f"Loaded {total} {suffix}. Click ▶ for a 30s highlight." ) self.player_reload_btn.config(state="normal") + if rows: + first = rows[0] + self._update_player_art( + first.get("path"), + title=first.get("title"), + artist=first.get("artist"), + ) def _on_player_tree_click(self, event) -> None: region = self.player_tree.identify_region(event.x, event.y) @@ -3164,8 +3230,24 @@ def _on_player_tree_click(self, event) -> None: return path = self.player_tree_paths.get(item) if path: + row = self.player_tree_rows.get(item, {}) + self._update_player_art( + path, title=row.get("title"), artist=row.get("artist") + ) self._play_preview(path, duration_ms=30000, player_item=item) + def _on_player_selection_change(self, _event) -> None: + if not hasattr(self, "player_tree"): + return + selection = self.player_tree.selection() + if not selection: + self._update_player_art(None) + return + item = selection[0] + path = self.player_tree_paths.get(item) + row = self.player_tree_rows.get(item, {}) + self._update_player_art(path, title=row.get("title"), artist=row.get("artist")) + def _send_help_query(self): threading.Thread(target=self._do_help_query, daemon=True).start() From 0ebca6d29634ffc36c934ddb41fa6d2f86867d2b Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 14 Dec 2025 08:24:27 -0500 Subject: [PATCH 023/606] Set HDBSCAN default samples and cluster size --- cluster_graph_panel.py | 2 +- clustered_playlists.py | 2 +- main_gui.py | 10 ++++++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/cluster_graph_panel.py b/cluster_graph_panel.py index 6cfdaf4..325be96 100644 --- a/cluster_graph_panel.py +++ b/cluster_graph_panel.py @@ -269,7 +269,7 @@ def open_param_dialog(self): dlg.grab_set() cs_var = tk.StringVar(value=str(self.cluster_params.get("min_cluster_size", 5))) - ms_var = tk.StringVar(value=str(self.cluster_params.get("min_samples", ""))) + ms_var = tk.StringVar(value=str(self.cluster_params.get("min_samples", 1))) eps_var = tk.StringVar(value=str(self.cluster_params.get("cluster_selection_epsilon", ""))) ttk.Label(dlg, text="Min cluster size:").grid(row=0, column=0, sticky="w") diff --git a/clustered_playlists.py b/clustered_playlists.py index b3e5193..a5ddb1f 100644 --- a/clustered_playlists.py +++ b/clustered_playlists.py @@ -62,7 +62,7 @@ def cluster_tracks( ) else: min_cluster_size = int(kwargs.get("min_cluster_size", 5)) - extra: dict = {} + extra: dict = {"min_samples": 1} if "min_samples" in kwargs: extra["min_samples"] = int(kwargs["min_samples"]) if "cluster_selection_epsilon" in kwargs: diff --git a/main_gui.py b/main_gui.py index 29c67ae..1d15b1f 100644 --- a/main_gui.py +++ b/main_gui.py @@ -161,11 +161,14 @@ def km_func(X, p): return HDBSCAN(**kwargs).fit_predict(X) min_cs = 5 - extras = {} + extras = {"min_samples": 1} if cluster_cfg and cluster_cfg.get("method") == "hdbscan": min_cs = int(cluster_cfg.get("min_cluster_size", cluster_cfg.get("num", 5))) + extras = {} if "min_samples" in cluster_cfg: extras["min_samples"] = int(cluster_cfg["min_samples"]) + else: + extras["min_samples"] = 1 if "cluster_selection_epsilon" in cluster_cfg: extras["cluster_selection_epsilon"] = float( cluster_cfg["cluster_selection_epsilon"] @@ -2004,7 +2007,10 @@ def _update_fields(*args): row=0, column=1, sticky="w", padx=(5, 0) ) ttk.Label(hdb_frame, text="Min samples:").grid(row=1, column=0, sticky="w") - min_samples_var = tk.StringVar(value="") + min_samples_default = "1" + if cluster_cfg and cluster_cfg.get("method") == "hdbscan" and "min_samples" in cluster_cfg: + min_samples_default = str(cluster_cfg["min_samples"]) + min_samples_var = tk.StringVar(value=min_samples_default) ttk.Entry(hdb_frame, textvariable=min_samples_var, width=10).grid( row=1, column=1, sticky="w", padx=(5, 0) ) From 5985fa6ca3ca151bfaeb8c500e224fd240c8dbfc Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 14 Dec 2025 08:25:42 -0500 Subject: [PATCH 024/606] Revert "Set HDBSCAN default samples and cluster size" --- cluster_graph_panel.py | 2 +- clustered_playlists.py | 2 +- main_gui.py | 10 ++-------- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/cluster_graph_panel.py b/cluster_graph_panel.py index 325be96..6cfdaf4 100644 --- a/cluster_graph_panel.py +++ b/cluster_graph_panel.py @@ -269,7 +269,7 @@ def open_param_dialog(self): dlg.grab_set() cs_var = tk.StringVar(value=str(self.cluster_params.get("min_cluster_size", 5))) - ms_var = tk.StringVar(value=str(self.cluster_params.get("min_samples", 1))) + ms_var = tk.StringVar(value=str(self.cluster_params.get("min_samples", ""))) eps_var = tk.StringVar(value=str(self.cluster_params.get("cluster_selection_epsilon", ""))) ttk.Label(dlg, text="Min cluster size:").grid(row=0, column=0, sticky="w") diff --git a/clustered_playlists.py b/clustered_playlists.py index a5ddb1f..b3e5193 100644 --- a/clustered_playlists.py +++ b/clustered_playlists.py @@ -62,7 +62,7 @@ def cluster_tracks( ) else: min_cluster_size = int(kwargs.get("min_cluster_size", 5)) - extra: dict = {"min_samples": 1} + extra: dict = {} if "min_samples" in kwargs: extra["min_samples"] = int(kwargs["min_samples"]) if "cluster_selection_epsilon" in kwargs: diff --git a/main_gui.py b/main_gui.py index 1d15b1f..29c67ae 100644 --- a/main_gui.py +++ b/main_gui.py @@ -161,14 +161,11 @@ def km_func(X, p): return HDBSCAN(**kwargs).fit_predict(X) min_cs = 5 - extras = {"min_samples": 1} + extras = {} if cluster_cfg and cluster_cfg.get("method") == "hdbscan": min_cs = int(cluster_cfg.get("min_cluster_size", cluster_cfg.get("num", 5))) - extras = {} if "min_samples" in cluster_cfg: extras["min_samples"] = int(cluster_cfg["min_samples"]) - else: - extras["min_samples"] = 1 if "cluster_selection_epsilon" in cluster_cfg: extras["cluster_selection_epsilon"] = float( cluster_cfg["cluster_selection_epsilon"] @@ -2007,10 +2004,7 @@ def _update_fields(*args): row=0, column=1, sticky="w", padx=(5, 0) ) ttk.Label(hdb_frame, text="Min samples:").grid(row=1, column=0, sticky="w") - min_samples_default = "1" - if cluster_cfg and cluster_cfg.get("method") == "hdbscan" and "min_samples" in cluster_cfg: - min_samples_default = str(cluster_cfg["min_samples"]) - min_samples_var = tk.StringVar(value=min_samples_default) + min_samples_var = tk.StringVar(value="") ttk.Entry(hdb_frame, textvariable=min_samples_var, width=10).grid( row=1, column=1, sticky="w", padx=(5, 0) ) From 92d35d10d2dd7b58971b59688c52bf6121aa8587 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 14 Dec 2025 08:34:40 -0500 Subject: [PATCH 025/606] Add live search to player tab --- main_gui.py | 61 +++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 48 insertions(+), 13 deletions(-) diff --git a/main_gui.py b/main_gui.py index 29c67ae..97c0e88 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1014,6 +1014,7 @@ def __init__(self): self.player_tree_rows: dict[str, dict[str, str]] = {} self._player_load_thread: threading.Thread | None = None self.player_art_image: ImageTk.PhotoImage | None = None + self.player_search_var = tk.StringVar(value="") # assume ffmpeg is available without performing checks self.ffmpeg_available = True @@ -1419,6 +1420,12 @@ def build_ui(self): ttk.Label(player_controls, textvariable=self.player_status_var).pack( side="left" ) + ttk.Label(player_controls, text="Search:").pack(side="left", padx=(10, 4)) + player_search = ttk.Entry( + player_controls, textvariable=self.player_search_var, width=30 + ) + player_search.pack(side="left") + self.player_search_var.trace_add("write", lambda *_: self._apply_player_filter()) self.player_reload_btn = ttk.Button( player_controls, text="Reload", @@ -1612,6 +1619,8 @@ def select_library(self): self.library_path_var.set(info["path"]) self.mapping_path = os.path.join(self.library_path, ".genre_mapping.json") self._load_genre_mapping() + if hasattr(self, "player_search_var"): + self.player_search_var.set("") # Clear any cached clustering data when switching libraries self.cluster_data = None if hasattr(self, "scan_btn"): @@ -3175,8 +3184,30 @@ def _load_player_library(self, library_path: str) -> None: def _update_player_table(self, rows: list[dict[str, str]], library_path: str) -> None: if library_path != self.library_path: return - self._clear_player_table() self.player_tracks = rows + self._apply_player_filter(total_count=len(rows)) + + def _apply_player_filter(self, *_args, total_count: int | None = None) -> None: + if not hasattr(self, "player_tree"): + return + query = self.player_search_var.get().strip().lower() + if query: + rows = [r for r in self.player_tracks if self._matches_player_query(r, query)] + else: + rows = list(self.player_tracks) + self._render_player_rows(rows, total_count=total_count) + + def _matches_player_query(self, row: dict[str, str], query: str) -> bool: + for key in ("title", "artist", "album", "length", "path"): + val = row.get(key) + if val and query in str(val).lower(): + return True + return False + + def _render_player_rows( + self, rows: list[dict[str, str]], total_count: int | None = None + ) -> None: + self._clear_player_table() play_icon = "▶" if self.preview_backend_available else "—" for row in rows: item = self.player_tree.insert( @@ -3192,18 +3223,7 @@ def _update_player_table(self, rows: list[dict[str, str]], library_path: str) -> ) self.player_tree_paths[item] = row["path"] self.player_tree_rows[item] = row - total = len(rows) - suffix = "track" if total == 1 else "tracks" - if not self.preview_backend_available: - reason = self.preview_backend_error or "Install python-vlc to enable playback." - self.player_status_var.set( - f"Loaded {total} {suffix}. Preview disabled: {reason}" - ) - else: - self.player_status_var.set( - f"Loaded {total} {suffix}. Click ▶ for a 30s highlight." - ) - self.player_reload_btn.config(state="normal") + if rows: first = rows[0] self._update_player_art( @@ -3211,6 +3231,21 @@ def _update_player_table(self, rows: list[dict[str, str]], library_path: str) -> title=first.get("title"), artist=first.get("artist"), ) + else: + self._update_player_art(None) + + total = total_count if total_count is not None else len(rows) + shown = len(rows) + suffix = "track" if total == 1 else "tracks" + if not self.preview_backend_available: + reason = self.preview_backend_error or "Install python-vlc to enable playback." + status = f"Loaded {total} {suffix}. Preview disabled: {reason}" + else: + status = f"Loaded {total} {suffix}. Click ▶ for a 30s highlight." + if shown != total: + status += f" Showing {shown} match{'es' if shown != 1 else ''}." + self.player_status_var.set(status) + self.player_reload_btn.config(state="normal") def _on_player_tree_click(self, event) -> None: region = self.player_tree.identify_region(event.x, event.y) From a1b9e8d5aa6bfabc11ef1485e6c40ba32f222ab6 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 14 Dec 2025 08:36:30 -0500 Subject: [PATCH 026/606] Improve cluster dialog defaults and guidance --- main_gui.py | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/main_gui.py b/main_gui.py index 29c67ae..24fce60 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1937,8 +1937,11 @@ def cluster_playlists_dialog(self, method: str = "kmeans"): dlg.grab_set() dlg.resizable(True, True) - method_var = tk.StringVar(value=method) - engine_var = tk.StringVar(value="") + cluster_cfg = getattr(self, "cluster_params", {}) or {} + selected_method = cluster_cfg.get("method", method) + + method_var = tk.StringVar(value=selected_method) + engine_var = tk.StringVar(value=cluster_cfg.get("engine", "librosa")) top = ttk.Frame(dlg) top.pack(fill="x", padx=10, pady=(10, 0)) @@ -1990,26 +1993,49 @@ def _update_fields(*args): # KMeans params km_frame = ttk.Frame(params_frame) - km_var = tk.StringVar(value="5") + km_var = tk.StringVar(value=str(cluster_cfg.get("n_clusters", 5))) ttk.Label(km_frame, text="Number of clusters:").pack(side="left") ttk.Entry(km_frame, textvariable=km_var, width=10).pack( side="left", padx=(5, 0) ) + ttk.Label( + km_frame, + text="(try ~5-20 for small libraries, 50-200 for large)", + foreground="gray", + ).pack(side="left", padx=(5, 0)) # HDBSCAN params hdb_frame = ttk.Frame(params_frame) - min_size_var = tk.StringVar(value="5") + min_size_var = tk.StringVar(value=str(cluster_cfg.get("min_cluster_size", 5))) ttk.Label(hdb_frame, text="Min cluster size:").grid(row=0, column=0, sticky="w") ttk.Entry(hdb_frame, textvariable=min_size_var, width=10).grid( row=0, column=1, sticky="w", padx=(5, 0) ) + ttk.Label( + hdb_frame, + text="(e.g., 5-15 for small sets, 30-80 for large)", + foreground="gray", + ).grid(row=0, column=2, sticky="w", padx=(5, 0)) ttk.Label(hdb_frame, text="Min samples:").grid(row=1, column=0, sticky="w") - min_samples_var = tk.StringVar(value="") + min_samples_var = tk.StringVar( + value=str(cluster_cfg.get("min_samples", "")) + if "min_samples" in cluster_cfg + else "1" + ) ttk.Entry(hdb_frame, textvariable=min_samples_var, width=10).grid( row=1, column=1, sticky="w", padx=(5, 0) ) + ttk.Label( + hdb_frame, + text="(start with 1-5; increase for stricter clusters)", + foreground="gray", + ).grid(row=1, column=2, sticky="w", padx=(5, 0)) ttk.Label(hdb_frame, text="Epsilon:").grid(row=2, column=0, sticky="w") - epsilon_var = tk.StringVar(value="") + epsilon_var = tk.StringVar( + value=str(cluster_cfg.get("cluster_selection_epsilon", "")) + if "cluster_selection_epsilon" in cluster_cfg + else "" + ) ttk.Entry(hdb_frame, textvariable=epsilon_var, width=10).grid( row=2, column=1, sticky="w", padx=(5, 0) ) From fb965b6e0a5bddf2da87d4f8e5836ba77d964e1a Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 14 Dec 2025 16:09:11 -0500 Subject: [PATCH 027/606] Add parallel processing option for clustered playlists --- clustered_playlists.py | 97 +++++++++++++++++++++++++------ controllers/cluster_controller.py | 2 +- main_gui.py | 26 +++++---- 3 files changed, 93 insertions(+), 32 deletions(-) diff --git a/clustered_playlists.py b/clustered_playlists.py index b3e5193..c441d24 100644 --- a/clustered_playlists.py +++ b/clustered_playlists.py @@ -1,10 +1,11 @@ import os +from concurrent.futures import ProcessPoolExecutor, as_completed +import multiprocessing import numpy as np import librosa from sklearn.cluster import KMeans from sklearn.preprocessing import StandardScaler import hdbscan -from playlist_generator import DEFAULT_EXTS def _ensure_1d(a): @@ -42,10 +43,72 @@ def extract_audio_features(file_path: str, log_callback=None) -> np.ndarray: return vec +def _extract_worker(path: str) -> tuple[str, np.ndarray, str | None]: + """Process-pool friendly wrapper around :func:`extract_audio_features`.""" + + try: + return path, extract_audio_features(path), None + except Exception as exc: # pragma: no cover - defensive logging path + return path, np.zeros(27, dtype=np.float32), str(exc) + + +def _extract_features_parallel(tracks, cache, log_callback): + """Fill ``cache`` for ``tracks`` using a bounded process pool.""" + + feats: list[np.ndarray] = [] + missing = [p for p in tracks if p not in cache] + + if missing: + workers = max(1, min(multiprocessing.cpu_count(), 4)) + log_callback( + f"⚙ Parallel feature extraction for {len(missing)} files ({workers} workers)" + ) + with ProcessPoolExecutor(max_workers=workers) as ex: + future_map = {ex.submit(_extract_worker, path): path for path in missing} + for done, fut in enumerate(as_completed(future_map), 1): + path, vec, err = fut.result() + if err: + log_callback(f"! Failed features for {path}: {err}") + else: + log_callback( + f"• Extracted features {done}/{len(missing)}: {os.path.basename(path)}" + ) + cache[path] = vec + + for path in tracks: + if path in cache: + log_callback(f"• Using cached features for {os.path.basename(path)}") + feats.append(cache[path]) + + return feats, bool(missing) + + +def _extract_features_serial(tracks, cache, log_callback): + """Sequential feature extraction (original behavior).""" + + feats: list[np.ndarray] = [] + updated = False + for idx, path in enumerate(tracks, 1): + if path in cache: + log_callback(f"• Using cached features for {os.path.basename(path)}") + else: + log_callback(f"• Extracting features {idx}/{len(tracks)}") + try: + cache[path] = extract_audio_features(path, log_callback) + except Exception as e: # pragma: no cover - defensive path + log_callback(f"! Failed features for {path}: {e}") + cache[path] = np.zeros(27, dtype=np.float32) + updated = True + feats.append(cache[path]) + + return feats, updated + + def cluster_tracks( feature_matrix: np.ndarray, method: str = "kmeans", log_callback=None, + engine: str = "serial", **kwargs, ) -> np.ndarray: """Cluster a matrix of feature vectors using KMeans or HDBSCAN.""" @@ -57,7 +120,7 @@ def cluster_tracks( log_callback( f"⚙ Clustering {len(feature_matrix)} tracks into {n_clusters} groups …" ) - labels = KMeans(n_clusters=n_clusters, random_state=0).fit_predict( + labels = KMeans(n_clusters=n_clusters, random_state=0, n_init="auto").fit_predict( feature_matrix ) else: @@ -69,6 +132,8 @@ def cluster_tracks( extra["cluster_selection_epsilon"] = float( kwargs["cluster_selection_epsilon"] ) + if engine == "parallel": + extra["core_dist_n_jobs"] = max(1, multiprocessing.cpu_count() - 1) log_callback("⚙ Running clustering algorithm …") labels = hdbscan.HDBSCAN( min_cluster_size=min_cluster_size, **extra @@ -84,7 +149,7 @@ def generate_clustered_playlists( method: str, params: dict, log_callback=None, - engine: str = "librosa", + engine: str = "serial", ) -> None: """Create clustered playlists for the given tracks.""" if log_callback is None: @@ -104,22 +169,14 @@ def generate_clustered_playlists( cache = {} log_callback("→ No feature cache found; extracting all tracks") - log_callback(f"⚙ Extracting audio features with {engine}…") + engine_mode = "serial" if engine in (None, "librosa", "serial") else "parallel" - feats = [] - updated = False - for idx, path in enumerate(tracks, 1): - if path in cache: - log_callback(f"• Using cached features for {os.path.basename(path)}") - else: - log_callback(f"• Extracting features {idx}/{len(tracks)}") - try: - cache[path] = extract_audio_features(path, log_callback) - except Exception as e: - log_callback(f"! Failed features for {path}: {e}") - cache[path] = np.zeros(27, dtype=np.float32) - updated = True - feats.append(cache[path]) + log_callback(f"⚙ Extracting audio features with {engine_mode} engine …") + + if engine_mode == "parallel": + feats, updated = _extract_features_parallel(tracks, cache, log_callback) + else: + feats, updated = _extract_features_serial(tracks, cache, log_callback) if updated: np.save(cache_file, cache) @@ -128,7 +185,9 @@ def generate_clustered_playlists( X = np.vstack(feats) X = StandardScaler().fit_transform(X) - labels = cluster_tracks(X, method, log_callback=log_callback, **params) + labels = cluster_tracks( + X, method, log_callback=log_callback, engine=engine_mode, **params + ) log_callback( f"✓ Clustering complete: found {len(set([l for l in labels if l >= 0]) )} clusters" ) diff --git a/controllers/cluster_controller.py b/controllers/cluster_controller.py index 8733291..fb17e84 100644 --- a/controllers/cluster_controller.py +++ b/controllers/cluster_controller.py @@ -48,7 +48,7 @@ def cluster_library( cluster_params: dict, log_callback, folder_filter: dict | None = None, - engine: str = "librosa", + engine: str = "serial", ) -> tuple[list[str], list]: """Generate clustered playlists for ``library_path`` and return features.""" diff --git a/main_gui.py b/main_gui.py index 76175d2..82695e3 100644 --- a/main_gui.py +++ b/main_gui.py @@ -145,9 +145,9 @@ def km_func(X, p): n_clusters = 5 if cluster_cfg and cluster_cfg.get("method") == "kmeans": n_clusters = int(cluster_cfg.get("num", 5)) - engine = "librosa" + engine = "serial" if cluster_cfg and "engine" in cluster_cfg: - engine = cluster_cfg["engine"] + engine = "serial" if cluster_cfg["engine"] == "librosa" else cluster_cfg["engine"] params = {"n_clusters": n_clusters, "method": "kmeans", "engine": engine} elif name == "Interactive – HDBSCAN": from hdbscan import HDBSCAN @@ -170,9 +170,9 @@ def km_func(X, p): extras["cluster_selection_epsilon"] = float( cluster_cfg["cluster_selection_epsilon"] ) - engine = "librosa" + engine = "serial" if cluster_cfg and "engine" in cluster_cfg: - engine = cluster_cfg["engine"] + engine = "serial" if cluster_cfg["engine"] == "librosa" else cluster_cfg["engine"] params = { "min_cluster_size": min_cs, "method": "hdbscan", @@ -1950,7 +1950,10 @@ def cluster_playlists_dialog(self, method: str = "kmeans"): selected_method = cluster_cfg.get("method", method) method_var = tk.StringVar(value=selected_method) - engine_var = tk.StringVar(value=cluster_cfg.get("engine", "librosa")) + engine_default = cluster_cfg.get("engine", "serial") + if engine_default == "librosa": + engine_default = "serial" + engine_var = tk.StringVar(value=engine_default) top = ttk.Frame(dlg) top.pack(fill="x", padx=10, pady=(10, 0)) @@ -1983,21 +1986,20 @@ def _update_fields(*args): params_frame = ttk.Frame(dlg) params_frame.pack(fill="x", padx=10, pady=(5, 0)) - engine_frame = ttk.LabelFrame(dlg, text="Feature Extraction Engine") + engine_frame = ttk.LabelFrame(dlg, text="Processing Engine") engine_frame.pack(fill="x", padx=10, pady=(10, 0)) ttk.Radiobutton( engine_frame, - text="Librosa (current)", + text="Standard (single process)", variable=engine_var, - value="librosa", + value="serial", ).pack(anchor="w", padx=5, pady=(2, 0)) ttk.Radiobutton( engine_frame, - text="Essentia (coming soon)", + text="Parallel (multi-core)", variable=engine_var, - value="essentia", - state="disabled", + value="parallel", ).pack(anchor="w", padx=5, pady=(0, 5)) # KMeans params @@ -2135,7 +2137,7 @@ def generate(): engine = engine_var.get() if not engine: messagebox.showerror( - "Select Engine", "Please select a feature extraction engine." + "Select Engine", "Please select a processing engine." ) return From a788584e3bf95c35611f909a4bb8d6c794f5701b Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 14 Dec 2025 17:03:09 -0500 Subject: [PATCH 028/606] Reduce HDBSCAN panel startup lag --- cluster_graph_panel.py | 62 +++++++++++++++++++++++++++++++++++++----- main_gui.py | 1 + 2 files changed, 56 insertions(+), 7 deletions(-) diff --git a/cluster_graph_panel.py b/cluster_graph_panel.py index 6cfdaf4..1e03574 100644 --- a/cluster_graph_panel.py +++ b/cluster_graph_panel.py @@ -1,4 +1,5 @@ import os +import threading from datetime import datetime import numpy as np @@ -40,8 +41,8 @@ def __init__( from sklearn.decomposition import PCA - X = np.vstack(features) - self.X2 = PCA(n_components=2).fit_transform(X) + self.X = np.vstack(features) + self.X2 = PCA(n_components=2).fit_transform(self.X) fig = Figure(figsize=(5, 5)) self.ax = fig.add_subplot(111) @@ -49,7 +50,12 @@ def __init__( self.canvas.get_tk_widget().pack(fill="both", expand=True) self.scatter = None - self._draw_clusters(cluster_func(X, cluster_params)) + self._loading_var = tk.StringVar(value="Running clustering …") + self._loading_lbl = ttk.Label(self, textvariable=self._loading_var) + self._loading_lbl.pack(pady=10) + self._clusters_ready = False + self._cluster_thread: threading.Thread | None = None + self._start_clustering(cluster_params) self.lasso = None self.sel_scatter = None @@ -71,6 +77,27 @@ def __init__( ok_btn: ttk.Button gen_btn: ttk.Button + def _start_clustering(self, params: dict): + """Kick off clustering work in a background thread.""" + + def _worker(): + try: + labels = self.cluster_func(self.X, params) + except Exception as exc: # pragma: no cover - defensive path for GUI + self.after(0, lambda: self.log(f"\u2717 Clustering failed: {exc}")) + return + self.after(0, lambda: self._on_clusters_ready(labels, params)) + + if self._cluster_thread and self._cluster_thread.is_alive(): + return + + self._clusters_ready = False + self._sync_controls() + self._loading_var.set("Running clustering …") + self._loading_lbl.pack(pady=10) + self._cluster_thread = threading.Thread(target=_worker, daemon=True) + self._cluster_thread.start() + # ─── Lasso Handling ───────────────────────────────────────────────────── def toggle_lasso(self): """Enter or exit lasso selection mode.""" @@ -252,15 +279,36 @@ def _draw_clusters(self, labels): self.ax.set_title("Lasso to select & generate playlist") self.canvas.draw_idle() + def _on_clusters_ready(self, labels, params: dict): + self.cluster_params = params + self._clusters_ready = True + if self._loading_lbl.winfo_exists(): + self._loading_lbl.pack_forget() + self._draw_clusters(labels) + self._sync_controls() + def recluster(self, params: dict): """Re-run clustering with new ``params`` and redraw the plot.""" - self.cluster_params = params if self.lasso is not None: self.toggle_lasso() self._clear_selection() - X = np.vstack(self.features) - labels = self.cluster_func(X, params) - self._draw_clusters(labels) + self._loading_var.set("Updating clusters …") + self._loading_lbl.pack(pady=10) + self._start_clustering(params) + + def _sync_controls(self): + """Enable/disable controls based on clustering readiness.""" + state = "normal" if self._clusters_ready else "disabled" + if hasattr(self, "lasso_btn"): + self.lasso_btn.configure(state=state) + if not self._clusters_ready: + for attr in ("ok_btn", "gen_btn"): + if hasattr(self, attr): + getattr(self, attr).configure(state="disabled") + + def refresh_control_states(self): + """Public hook to sync controls after external widgets are attached.""" + self._sync_controls() def open_param_dialog(self): """Show dialog to edit HDBSCAN parameters and redraw clusters.""" diff --git a/main_gui.py b/main_gui.py index 82695e3..c7208b9 100644 --- a/main_gui.py +++ b/main_gui.py @@ -813,6 +813,7 @@ def _auto_create_all(): hover_panel.place_forget() panel.setup_hover(hover_panel, art_lbl, title_lbl, artist_lbl) + panel.refresh_control_states() return frame From 113e467befade2935f4ce00f0c2c9981f3774393 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 14 Dec 2025 17:03:31 -0500 Subject: [PATCH 029/606] Add return payload for tempo/energy bucket tool --- playlist_engine.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/playlist_engine.py b/playlist_engine.py index 4028654..0e2c0e6 100644 --- a/playlist_engine.py +++ b/playlist_engine.py @@ -107,6 +107,15 @@ def bucket_by_tempo_energy( for (tb, eb), items in buckets.items() } + return { + "buckets": buckets, + "playlist_paths": out_paths, + "stats": stats, + "processed": processed, + "total": len(tracks), + "cancelled": bool(cancel_event and cancel_event.is_set()), + } + def _split_genres(genres: Iterable[str]) -> list[str]: parts: list[str] = [] From 3e9c8397cddc87701d4a333faba3950446d8755b Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 14 Dec 2025 17:15:32 -0500 Subject: [PATCH 030/606] Fix cluster settings reuse for HDBSCAN switch --- main_gui.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/main_gui.py b/main_gui.py index c7208b9..2951caa 100644 --- a/main_gui.py +++ b/main_gui.py @@ -144,7 +144,7 @@ def km_func(X, p): n_clusters = 5 if cluster_cfg and cluster_cfg.get("method") == "kmeans": - n_clusters = int(cluster_cfg.get("num", 5)) + n_clusters = int(cluster_cfg.get("n_clusters", 5)) engine = "serial" if cluster_cfg and "engine" in cluster_cfg: engine = "serial" if cluster_cfg["engine"] == "librosa" else cluster_cfg["engine"] @@ -163,7 +163,9 @@ def km_func(X, p): min_cs = 5 extras = {} if cluster_cfg and cluster_cfg.get("method") == "hdbscan": - min_cs = int(cluster_cfg.get("min_cluster_size", cluster_cfg.get("num", 5))) + min_cs = int( + cluster_cfg.get("min_cluster_size", cluster_cfg.get("n_clusters", 5)) + ) if "min_samples" in cluster_cfg: extras["min_samples"] = int(cluster_cfg["min_samples"]) if "cluster_selection_epsilon" in cluster_cfg: From 890f6b5de1d7a57283fcf812c4c4d59c796d0a0f Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 14 Dec 2025 19:18:20 -0500 Subject: [PATCH 031/606] Cache interactive cluster panels --- cluster_graph_panel.py | 61 +++++++++++----- controllers/cluster_view_controller.py | 98 ++++++++++++++++++++++++++ main_gui.py | 66 ++++++++++++++--- 3 files changed, 200 insertions(+), 25 deletions(-) create mode 100644 controllers/cluster_view_controller.py diff --git a/cluster_graph_panel.py b/cluster_graph_panel.py index 1e03574..dbefaad 100644 --- a/cluster_graph_panel.py +++ b/cluster_graph_panel.py @@ -3,6 +3,8 @@ from datetime import datetime import numpy as np +import time +import logging import tkinter as tk from tkinter import ttk, messagebox from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg @@ -17,6 +19,8 @@ from mutagen import File as MutagenFile from music_indexer_api import get_tags +logger = logging.getLogger(__name__) + class ClusterGraphPanel(ttk.Frame): """Interactive scatter plot with lasso selection for playlist creation.""" @@ -25,24 +29,25 @@ def __init__( self, parent, tracks, - features, + X: np.ndarray, + X2: np.ndarray, cluster_func, cluster_params, + cluster_manager, + algo_key: str, library_path, log_callback, ): super().__init__(parent) self.tracks = tracks - self.features = features + self.X = X + self.X2 = X2 self.cluster_func = cluster_func self.cluster_params = cluster_params self.library_path = library_path self.log = log_callback - - from sklearn.decomposition import PCA - - self.X = np.vstack(features) - self.X2 = PCA(n_components=2).fit_transform(self.X) + self.cluster_manager = cluster_manager + self.algo_key = algo_key fig = Figure(figsize=(5, 5)) self.ax = fig.add_subplot(111) @@ -55,6 +60,7 @@ def __init__( self._loading_lbl.pack(pady=10) self._clusters_ready = False self._cluster_thread: threading.Thread | None = None + self._cluster_start_ts: float | None = None self._start_clustering(cluster_params) self.lasso = None @@ -82,7 +88,13 @@ def _start_clustering(self, params: dict): def _worker(): try: - labels = self.cluster_func(self.X, params) + cached = self.cluster_manager.get_cached_labels(self.algo_key, params) + if cached is None: + labels = self.cluster_manager.compute_labels( + self.algo_key, params, self.cluster_func + ) + else: + labels = cached except Exception as exc: # pragma: no cover - defensive path for GUI self.after(0, lambda: self.log(f"\u2717 Clustering failed: {exc}")) return @@ -91,10 +103,17 @@ def _worker(): if self._cluster_thread and self._cluster_thread.is_alive(): return + cached = self.cluster_manager.get_cached_labels(self.algo_key, params) + if cached is not None: + self._on_clusters_ready(cached, params) + return + self._clusters_ready = False self._sync_controls() self._loading_var.set("Running clustering …") self._loading_lbl.pack(pady=10) + self._cluster_start_ts = time.perf_counter() + logger.info("[perf] start clustering %s params=%s", self.algo_key, params) self._cluster_thread = threading.Thread(target=_worker, daemon=True) self._cluster_thread.start() @@ -269,22 +288,32 @@ def _compute_colors(self, labels): def _draw_clusters(self, labels): point_colors, self.label_colors = self._compute_colors(labels) self.labels = labels - self.ax.clear() - self.scatter = self.ax.scatter( - self.X2[:, 0], - self.X2[:, 1], - c=point_colors, - s=20, - ) - self.ax.set_title("Lasso to select & generate playlist") + if self.scatter is None: + self.scatter = self.ax.scatter( + self.X2[:, 0], + self.X2[:, 1], + c=point_colors, + s=20, + ) + self.ax.set_title("Lasso to select & generate playlist") + else: + self.scatter.set_offsets(self.X2) + self.scatter.set_color(point_colors) self.canvas.draw_idle() def _on_clusters_ready(self, labels, params: dict): + duration_ms = None + if self._cluster_start_ts is not None: + duration_ms = (time.perf_counter() - self._cluster_start_ts) * 1000 self.cluster_params = params self._clusters_ready = True if self._loading_lbl.winfo_exists(): self._loading_lbl.pack_forget() self._draw_clusters(labels) + if duration_ms is not None: + logger.info( + "[perf] clusters ready (%s) in %.1f ms", self.algo_key, duration_ms + ) self._sync_controls() def recluster(self, params: dict): diff --git a/controllers/cluster_view_controller.py b/controllers/cluster_view_controller.py new file mode 100644 index 0000000..055f1e9 --- /dev/null +++ b/controllers/cluster_view_controller.py @@ -0,0 +1,98 @@ +"""Caching helpers for interactive clustering views.""" + +from __future__ import annotations + +import hashlib +import logging +import threading +import time +from typing import Callable + +import numpy as np +from sklearn.decomposition import PCA + +logger = logging.getLogger(__name__) + + +class ClusterComputationManager: + """Cache PCA projections and clustering labels for interactive panels.""" + + def __init__(self, tracks: list[str], features: list, log_callback: Callable[[str], None]): + self.tracks = tracks + self.features = features + self.log_callback = log_callback + + self._X: np.ndarray | None = None + self._X_key: str | None = None + self._pca_cache: dict[tuple[str, tuple], np.ndarray] = {} + self._labels_cache: dict[tuple[str, str, tuple], np.ndarray] = {} + self._lock = threading.Lock() + + # ─── Dataset Helpers ────────────────────────────────────────────────── + def _materialize_features(self) -> np.ndarray: + if self._X is not None: + return self._X + + start = time.perf_counter() + self._X = np.vstack(self.features) + duration = (time.perf_counter() - start) * 1000 + logger.info("[perf] materialize features in %.1f ms", duration) + return self._X + + def dataset_key(self) -> str: + if self._X_key: + return self._X_key + + X = self._materialize_features().astype(np.float32, copy=False) + start = time.perf_counter() + digest = hashlib.sha1(X.tobytes()).hexdigest() + duration = (time.perf_counter() - start) * 1000 + self._X_key = digest + logger.info("[perf] dataset fingerprint in %.1f ms", duration) + return digest + + # ─── PCA Caching ────────────────────────────────────────────────────── + def get_projection(self, pca_params: tuple = (2,)) -> tuple[np.ndarray, np.ndarray]: + """Return (X, X_2d) using PCA with caching.""" + + key = (self.dataset_key(), pca_params) + with self._lock: + if key in self._pca_cache: + logger.info("[perf] PCA cache hit for key=%s", key) + return self._materialize_features(), self._pca_cache[key] + + start = time.perf_counter() + X = self._materialize_features() + pca = PCA(n_components=pca_params[0]) + X2 = pca.fit_transform(X) + duration = (time.perf_counter() - start) * 1000 + logger.info("[perf] PCA compute in %.1f ms", duration) + + with self._lock: + self._pca_cache[key] = X2 + return X, X2 + + # ─── Clustering Cache ───────────────────────────────────────────────── + def get_cached_labels(self, algo: str, params: dict) -> np.ndarray | None: + key = (self.dataset_key(), algo, tuple(sorted(params.items()))) + with self._lock: + labels = self._labels_cache.get(key) + if labels is not None: + logger.info("[perf] clustering cache hit for %s params=%s", algo, params) + return labels + + def compute_labels(self, algo: str, params: dict, cluster_func: Callable) -> np.ndarray: + key = (self.dataset_key(), algo, tuple(sorted(params.items()))) + with self._lock: + if key in self._labels_cache: + logger.info("[perf] clustering cache hit for %s params=%s", algo, params) + return self._labels_cache[key] + + start = time.perf_counter() + labels = cluster_func(self._materialize_features(), params) + duration = (time.perf_counter() - start) * 1000 + logger.info("[perf] clustering compute for %s in %.1f ms", algo, duration) + + with self._lock: + self._labels_cache[key] = labels + return labels diff --git a/main_gui.py b/main_gui.py index 2951caa..a5b89ef 100644 --- a/main_gui.py +++ b/main_gui.py @@ -76,6 +76,7 @@ ) from plugins.assistant_plugin import AssistantPlugin from controllers.cluster_controller import cluster_library +from controllers.cluster_view_controller import ClusterComputationManager from config import load_config, save_config, DEFAULT_FP_THRESHOLDS import playlist_engine from playlist_engine import bucket_by_tempo_energy, autodj_playlist @@ -131,10 +132,14 @@ def create_panel_for_plugin(app, name: str, parent: tk.Widget) -> ttk.Frame: cluster_data = getattr(app, "cluster_data", None) cluster_cfg = getattr(app, "cluster_params", None) + cluster_manager = getattr(app, "cluster_manager", None) if cluster_data is None: tracks = features = None else: tracks, features = cluster_data + if cluster_manager is None or getattr(cluster_manager, "tracks", None) != tracks: + app.cluster_manager = ClusterComputationManager(tracks, features, app._log) + cluster_manager = app.cluster_manager if name == "Interactive – KMeans": from sklearn.cluster import KMeans @@ -149,6 +154,7 @@ def km_func(X, p): if cluster_cfg and "engine" in cluster_cfg: engine = "serial" if cluster_cfg["engine"] == "librosa" else cluster_cfg["engine"] params = {"n_clusters": n_clusters, "method": "kmeans", "engine": engine} + algo_key = "kmeans" elif name == "Interactive – HDBSCAN": from hdbscan import HDBSCAN @@ -181,6 +187,7 @@ def km_func(X, p): "engine": engine, **extras, } + algo_key = "hdbscan" elif name == "Sort by Genre": lib_var = tk.StringVar(value=app.library_path or "No library selected") progress_var = tk.StringVar(value="Waiting to start") @@ -732,12 +739,20 @@ def generate(): container.rowconfigure(0, weight=1) container.columnconfigure(0, weight=1) + if cluster_manager is None: + return frame + + X, X2 = cluster_manager.get_projection() + panel = ClusterGraphPanel( container, tracks, - features, + X, + X2, cluster_func=km_func, cluster_params=params, + cluster_manager=cluster_manager, + algo_key=algo_key, library_path=app.library_path, log_callback=app._log, ) @@ -958,8 +973,13 @@ def __init__(self): # Cached tracks and feature vectors for interactive clustering self.cluster_data = None + self.cluster_manager = None self.folder_filter = {"include": [], "exclude": []} + # Cached plugin panels to avoid teardown/rebuild churn + self.plugin_views: dict[str, ttk.Frame] = {} + self.active_plugin: str | None = None + # Library Sync state self.sync_debug_var = tk.BooleanVar(value=False) self.sync_library_var = tk.StringVar(value="") @@ -1586,15 +1606,28 @@ def on_scale_change(self, event=None): def on_plugin_select(self, event): """Swap in the UI panel for the selected playlist plugin.""" - for w in self.plugin_panel.winfo_children(): - w.destroy() + switch_start = time.perf_counter() try: sel = self.plugin_list.get(self.plugin_list.curselection()) except tk.TclError: return - panel = create_panel_for_plugin(self, sel, parent=self.plugin_panel) + logging.info("[perf] tool switch -> %s", sel) + + if self.active_plugin and self.active_plugin in self.plugin_views: + logging.info("[perf] hide panel %s", self.active_plugin) + self.plugin_views[self.active_plugin].pack_forget() + + panel = self.plugin_views.get(sel) + if panel is None: + panel = create_panel_for_plugin(self, sel, parent=self.plugin_panel) + if panel: + self.plugin_views[sel] = panel if panel: + logging.info("[perf] show panel %s", sel) panel.pack(fill="both", expand=True) + self.active_plugin = sel + duration = (time.perf_counter() - switch_start) * 1000 + logging.info("[perf] tool switch complete in %.1f ms", duration) def _load_genre_mapping(self): """Load genre mapping from ``self.mapping_path`` if possible.""" @@ -1626,6 +1659,9 @@ def select_library(self): self.player_search_var.set("") # Clear any cached clustering data when switching libraries self.cluster_data = None + self.cluster_manager = None + self.plugin_views.clear() + self.active_plugin = None if hasattr(self, "scan_btn"): self.scan_btn.config(state="normal") self._validate_threshold() @@ -2205,6 +2241,15 @@ def _run_cluster_generation( ) self.cluster_data = (tracks, feats) self.cluster_params = {"method": method, "engine": engine, **params} + self.cluster_manager = ClusterComputationManager(tracks, feats, self._log) + + for name in ("Interactive – KMeans", "Interactive – HDBSCAN"): + if name in self.plugin_views: + try: + self.plugin_views[name].destroy() + except Exception: + pass + self.plugin_views.pop(name, None) def done(): messagebox.showinfo("Clustered Playlists", "Generation complete") @@ -2220,11 +2265,14 @@ def _refresh_plugin_panel(self): sel = self.plugin_list.get(self.plugin_list.curselection()) except tk.TclError: return - for w in self.plugin_panel.winfo_children(): - w.destroy() - panel = create_panel_for_plugin(self, sel, parent=self.plugin_panel) - if panel: - panel.pack(fill="both", expand=True) + if sel in self.plugin_views: + try: + self.plugin_views[sel].destroy() + except Exception: + pass + self.plugin_views.pop(sel, None) + self.active_plugin = None + self.on_plugin_select(None) def _tagfix_filter_dialog(self): dlg = tk.Toplevel(self) From 4235f57b732aae8cdb1fbafe45b5bd0270a17e21 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 14 Dec 2025 19:40:49 -0500 Subject: [PATCH 032/606] Add tag fixer API connectivity indicator --- main_gui.py | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/main_gui.py b/main_gui.py index a5b89ef..dce2ea8 100644 --- a/main_gui.py +++ b/main_gui.py @@ -75,6 +75,7 @@ scan_raw_genres, ) from plugins.assistant_plugin import AssistantPlugin +from plugins.acoustid_plugin import AcoustIDService, MusicBrainzService from controllers.cluster_controller import cluster_library from controllers.cluster_view_controller import ClusterComputationManager from config import load_config, save_config, DEFAULT_FP_THRESHOLDS @@ -1000,6 +1001,7 @@ def __init__(self): self.tf_apply_genres = tk.BooleanVar(value=False) self.tagfix_db_path = "" self.tagfix_debug_var = tk.BooleanVar(value=False) + self.tagfix_api_status = tk.StringVar(value="") self.dup_debug_var = tk.BooleanVar(value=False) @@ -1310,6 +1312,13 @@ def build_ui(self): text="Verbose Debug", variable=self.tagfix_debug_var, ).pack(side="left", padx=(5, 0)) + api_status = ttk.Frame(opts) + api_status.pack(side="left", padx=(10, 0)) + self.tagfix_api_indicator = tk.Label(api_status, text="●", fg="#d68a00") + self.tagfix_api_indicator.pack(side="left") + ttk.Label(api_status, textvariable=self.tagfix_api_status).pack( + side="left", padx=(4, 0) + ) self.tagfix_progress = ttk.Progressbar( self.tagfix_tab, orient="horizontal", mode="determinate" @@ -1433,6 +1442,7 @@ def build_ui(self): if self.library_path_var.get(): self.scan_btn.config(state="normal") self._validate_threshold() + self._check_tagfix_api_status() # ─── Player Tab ──────────────────────────────────────────────── self.player_tab = ttk.Frame(self.notebook) @@ -2538,6 +2548,37 @@ def _browse_tagfix_folder(self): save_last_path(folder) self.tagfix_folder_var.set(folder) + def _check_tagfix_api_status(self) -> None: + cfg = load_config() + service = cfg.get("metadata_service", "AcoustID") + self.tagfix_api_indicator.configure(fg="#d68a00") + self.tagfix_api_status.set(f"{service}: Checking…") + + def worker() -> None: + ok, msg = self._test_metadata_service(service) + + def done() -> None: + if not self.winfo_exists(): + return + color = "green" if ok else "red" + label = "Connected" if ok else (msg or "Unavailable") + self.tagfix_api_indicator.configure(fg=color) + self.tagfix_api_status.set(f"{service}: {label}") + + self.after(0, done) + + threading.Thread(target=worker, daemon=True).start() + + def _test_metadata_service(self, service: str) -> tuple[bool, str]: + try: + if service == "MusicBrainz": + return MusicBrainzService().test_connection() + if service == "AcoustID": + return AcoustIDService().test_connection() + return False, "Not supported" + except Exception as exc: # pragma: no cover - defensive UI guard + return False, str(exc) + # ── Library Quality Helpers ─────────────────────────────────────── def _browse_dup_folder(self) -> None: @@ -3399,6 +3440,14 @@ def open_metadata_settings(self): frame = MetadataServiceConfigFrame(win) frame.pack(fill="both", expand=True, padx=10, pady=10) + win.bind( + "", + lambda e: ( + self._check_tagfix_api_status() + if e.widget is win and self.winfo_exists() + else None + ), + ) self._metadata_win = win def _on_exit(self) -> None: From ed6199aec77b676d558fb5a13fe426e924912341 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 14 Dec 2025 19:46:26 -0500 Subject: [PATCH 033/606] Add Essentia feature extraction engine options --- clustered_playlists.py | 140 +++++++++++++++++++++++++----- controllers/cluster_controller.py | 2 + playlist_engine.py | 46 ++++++++-- 3 files changed, 163 insertions(+), 25 deletions(-) diff --git a/clustered_playlists.py b/clustered_playlists.py index c441d24..7d74bca 100644 --- a/clustered_playlists.py +++ b/clustered_playlists.py @@ -1,29 +1,75 @@ +import importlib.util import os from concurrent.futures import ProcessPoolExecutor, as_completed import multiprocessing -import numpy as np -import librosa +from typing import Literal + from sklearn.cluster import KMeans from sklearn.preprocessing import StandardScaler import hdbscan +def _module_available(name: str) -> bool: + try: + return importlib.util.find_spec(name) is not None + except ValueError: # pragma: no cover - defensive for broken environments + return False + + +if _module_available("numpy"): + import numpy as np # type: ignore +else: # pragma: no cover - optional dependency + np = None # type: ignore + +if _module_available("librosa"): + import librosa # type: ignore +else: # pragma: no cover - optional dependency + librosa = None # type: ignore + +if _module_available("essentia"): + import essentia # type: ignore + from essentia.standard import MonoLoader, MusicExtractor +else: # pragma: no cover - optional dependency + essentia = None # type: ignore + MonoLoader = MusicExtractor = None # type: ignore + +AudioFeatureEngine = Literal["librosa", "essentia"] + + def _ensure_1d(a): """Return ``a`` flattened to 1-D for feature stacking.""" + if np is None: + raise RuntimeError("numpy required for audio feature extraction") a = np.asarray(a) if a.ndim > 1: a = a.ravel() return a -def extract_audio_features(file_path: str, log_callback=None) -> np.ndarray: - """Return a simple feature vector for ``file_path`` using librosa.""" +def extract_audio_features( + file_path: str, log_callback=None, engine: AudioFeatureEngine = "librosa" +) -> "np.ndarray": + """Return a simple feature vector for ``file_path`` using the requested engine.""" + if log_callback is None: log_callback = lambda msg: None + if np is None: + raise RuntimeError("numpy required for audio feature extraction") + + if engine == "librosa": + return _extract_audio_features_librosa(file_path, log_callback) + if engine == "essentia": + return _extract_audio_features_essentia(file_path, log_callback) + raise ValueError(f"Unknown feature extraction engine: {engine}") + + +def _extract_audio_features_librosa(file_path: str, log_callback) -> "np.ndarray": + if librosa is None: + raise RuntimeError("librosa required for feature extraction (engine='librosa')") + y, sr = librosa.load(file_path, sr=None, mono=True) mfcc = librosa.feature.mfcc(y=y, sr=sr, n_mfcc=13) - # Log raw MFCC shape for debugging log_callback( f" \u00b7 MFCC shape for {os.path.basename(file_path)}: {mfcc.shape}" ) @@ -33,9 +79,38 @@ def extract_audio_features(file_path: str, log_callback=None) -> np.ndarray: std_mfcc = _ensure_1d(np.std(mfcc, axis=1)) tempo_arr = _ensure_1d(np.array([tempo], dtype=np.float32)) + return _assemble_feature_vector(mean_mfcc, std_mfcc, tempo_arr) + + +def _extract_audio_features_essentia(file_path: str, log_callback) -> "np.ndarray": + if ( + essentia is None + or MonoLoader is None + or MusicExtractor is None + ): + raise RuntimeError("Essentia required for feature extraction (engine='essentia')") + + extractor = MusicExtractor( + lowlevelStats=["mean", "stdev"], + rhythmStats=["mean"], + numberMfccCoefficients=13, + ) + features = extractor(file_path) + + mfcc_mean = _ensure_1d(np.asarray(features["lowlevel.mfcc.mean"], dtype=np.float32)) + mfcc_std = _ensure_1d(np.asarray(features["lowlevel.mfcc.stdev"], dtype=np.float32)) + tempo_arr = _ensure_1d(np.array([features["rhythm.bpm"]], dtype=np.float32)) + + log_callback( + f" \u00b7 Essentia MFCC lengths for {os.path.basename(file_path)}: {mfcc_mean.shape}" + ) + + return _assemble_feature_vector(mfcc_mean, mfcc_std, tempo_arr) + + +def _assemble_feature_vector(mean_mfcc, std_mfcc, tempo_arr) -> "np.ndarray": vec = np.hstack([mean_mfcc, std_mfcc, tempo_arr]).astype(np.float32) - # Validate length and ensure we have exactly 27 values if vec.shape[0] != 27: raise RuntimeError( f"Feature vector has wrong length {vec.shape[0]}, expected 27" @@ -43,19 +118,20 @@ def extract_audio_features(file_path: str, log_callback=None) -> np.ndarray: return vec -def _extract_worker(path: str) -> tuple[str, np.ndarray, str | None]: +def _extract_worker(path: str, engine: AudioFeatureEngine) -> tuple[str, "np.ndarray", str | None]: """Process-pool friendly wrapper around :func:`extract_audio_features`.""" try: - return path, extract_audio_features(path), None + return path, extract_audio_features(path, engine=engine), None except Exception as exc: # pragma: no cover - defensive logging path - return path, np.zeros(27, dtype=np.float32), str(exc) + zeros = np.zeros(27, dtype=np.float32) if np is not None else None + return path, zeros, str(exc) -def _extract_features_parallel(tracks, cache, log_callback): +def _extract_features_parallel(tracks, cache, log_callback, engine: AudioFeatureEngine): """Fill ``cache`` for ``tracks`` using a bounded process pool.""" - feats: list[np.ndarray] = [] + feats: list["np.ndarray"] = [] missing = [p for p in tracks if p not in cache] if missing: @@ -64,7 +140,9 @@ def _extract_features_parallel(tracks, cache, log_callback): f"⚙ Parallel feature extraction for {len(missing)} files ({workers} workers)" ) with ProcessPoolExecutor(max_workers=workers) as ex: - future_map = {ex.submit(_extract_worker, path): path for path in missing} + future_map = { + ex.submit(_extract_worker, path, engine): path for path in missing + } for done, fut in enumerate(as_completed(future_map), 1): path, vec, err = fut.result() if err: @@ -83,10 +161,10 @@ def _extract_features_parallel(tracks, cache, log_callback): return feats, bool(missing) -def _extract_features_serial(tracks, cache, log_callback): +def _extract_features_serial(tracks, cache, log_callback, engine: AudioFeatureEngine): """Sequential feature extraction (original behavior).""" - feats: list[np.ndarray] = [] + feats: list["np.ndarray"] = [] updated = False for idx, path in enumerate(tracks, 1): if path in cache: @@ -94,7 +172,7 @@ def _extract_features_serial(tracks, cache, log_callback): else: log_callback(f"• Extracting features {idx}/{len(tracks)}") try: - cache[path] = extract_audio_features(path, log_callback) + cache[path] = extract_audio_features(path, log_callback, engine=engine) except Exception as e: # pragma: no cover - defensive path log_callback(f"! Failed features for {path}: {e}") cache[path] = np.zeros(27, dtype=np.float32) @@ -105,12 +183,12 @@ def _extract_features_serial(tracks, cache, log_callback): def cluster_tracks( - feature_matrix: np.ndarray, + feature_matrix: "np.ndarray", method: str = "kmeans", log_callback=None, engine: str = "serial", **kwargs, -) -> np.ndarray: +) -> "np.ndarray": """Cluster a matrix of feature vectors using KMeans or HDBSCAN.""" if log_callback is None: log_callback = lambda msg: None @@ -150,11 +228,27 @@ def generate_clustered_playlists( params: dict, log_callback=None, engine: str = "serial", + feature_engine: AudioFeatureEngine = "librosa", ) -> None: """Create clustered playlists for the given tracks.""" if log_callback is None: log_callback = lambda msg: None + if np is None: + raise RuntimeError( + "numpy required for clustered playlist feature extraction and clustering" + ) + if feature_engine == "librosa" and librosa is None: + raise RuntimeError( + "librosa required for feature extraction (engine='librosa')" + ) + if feature_engine == "essentia" and ( + essentia is None or MonoLoader is None or MusicExtractor is None + ): + raise RuntimeError( + "Essentia required for feature extraction (engine='essentia')" + ) + # ------------------------------------------------------------------ # Feature cache setup # ------------------------------------------------------------------ @@ -171,12 +265,18 @@ def generate_clustered_playlists( engine_mode = "serial" if engine in (None, "librosa", "serial") else "parallel" - log_callback(f"⚙ Extracting audio features with {engine_mode} engine …") + log_callback( + f"⚙ Extracting audio features with {engine_mode} engine ({feature_engine}) …" + ) if engine_mode == "parallel": - feats, updated = _extract_features_parallel(tracks, cache, log_callback) + feats, updated = _extract_features_parallel( + tracks, cache, log_callback, feature_engine + ) else: - feats, updated = _extract_features_serial(tracks, cache, log_callback) + feats, updated = _extract_features_serial( + tracks, cache, log_callback, feature_engine + ) if updated: np.save(cache_file, cache) diff --git a/controllers/cluster_controller.py b/controllers/cluster_controller.py index fb17e84..98cc6f7 100644 --- a/controllers/cluster_controller.py +++ b/controllers/cluster_controller.py @@ -49,6 +49,7 @@ def cluster_library( log_callback, folder_filter: dict | None = None, engine: str = "serial", + feature_engine: str = "librosa", ) -> tuple[list[str], list]: """Generate clustered playlists for ``library_path`` and return features.""" @@ -71,5 +72,6 @@ def log(msg: str) -> None: cluster_params, log, engine=engine, + feature_engine=feature_engine, ) return tracks, feats diff --git a/playlist_engine.py b/playlist_engine.py index 0e2c0e6..23b9359 100644 --- a/playlist_engine.py +++ b/playlist_engine.py @@ -2,7 +2,7 @@ import os import math import re -from typing import Callable, Iterable +from typing import Callable, Iterable, Literal try: _mutagen_available = importlib.util.find_spec("mutagen") is not None @@ -35,6 +35,17 @@ class FLAC(_DummyAudio): except Exception: # pragma: no cover - librosa optional librosa = None +try: + _essentia_available = importlib.util.find_spec("essentia") is not None +except ValueError: # pragma: no cover - defensive for broken environments + _essentia_available = False +if _essentia_available: + import essentia # type: ignore + from essentia.standard import MonoLoader, RhythmExtractor2013 +else: # pragma: no cover - optional dependency + essentia = None + MonoLoader = RhythmExtractor2013 = None # type: ignore + from playlist_generator import write_playlist, DEFAULT_EXTS @@ -54,30 +65,55 @@ def categorize_energy(rms: float) -> str: return "high" -def compute_tempo_energy(path: str) -> tuple[float, float]: +TempoEngine = Literal["librosa", "essentia"] + + +def compute_tempo_energy(path: str, engine: TempoEngine = "librosa") -> tuple[float, float]: + if engine == "librosa": + return _compute_tempo_energy_librosa(path) + if engine == "essentia": + return _compute_tempo_energy_essentia(path) + raise ValueError(f"Unknown tempo engine: {engine}") + + +def _compute_tempo_energy_librosa(path: str) -> tuple[float, float]: if librosa is None or np is None: - raise RuntimeError("librosa/numpy required for tempo analysis") + raise RuntimeError("librosa/numpy required for tempo analysis (engine='librosa')") y, sr = librosa.load(path, mono=True, sr=None) tempo, _ = librosa.beat.beat_track(y=y, sr=sr) rms = float(np.mean(librosa.feature.rms(y=y))) return tempo, rms +def _compute_tempo_energy_essentia(path: str) -> tuple[float, float]: + if essentia is None or np is None or MonoLoader is None or RhythmExtractor2013 is None: + raise RuntimeError("Essentia/numpy required for tempo analysis (engine='essentia')") + audio = np.asarray(MonoLoader(filename=path)(), dtype=np.float32) + tempo, *_ = RhythmExtractor2013(method="multifeature")(audio) + rms = float(np.sqrt(np.mean(np.square(audio)))) if audio.size else 0.0 + return float(tempo), rms + + def bucket_by_tempo_energy( tracks: list[str], root_path: str, log_callback=None, progress_callback=None, cancel_event=None, + engine: TempoEngine = "librosa", ) -> dict: if log_callback is None: log_callback = lambda m: None if progress_callback is None: progress_callback = lambda _count: None - if librosa is None or np is None: + if engine == "librosa" and (librosa is None or np is None): raise RuntimeError( "Tempo/Energy buckets require numpy and librosa. Install with `pip install -r requirements.txt`." ) + if engine == "essentia" and (essentia is None or np is None): + raise RuntimeError( + "Tempo/Energy buckets require numpy and Essentia when engine='essentia'. Install Essentia to continue." + ) playlists_dir = os.path.join(root_path, "Playlists") os.makedirs(playlists_dir, exist_ok=True) buckets: dict[tuple[str, str], list[str]] = {} @@ -87,7 +123,7 @@ def bucket_by_tempo_energy( log_callback("! Bucket generation cancelled by user.") break try: - tempo, rms = compute_tempo_energy(path) + tempo, rms = compute_tempo_energy(path, engine=engine) tb = categorize_tempo(tempo) eb = categorize_energy(rms) buckets.setdefault((tb, eb), []).append(path) From 5607c89ba98178ff5dc3317a282c54f61b4f9dfa Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 14 Dec 2025 19:51:34 -0500 Subject: [PATCH 034/606] Add engine selection sharing for tempo buckets and Auto-DJ --- main_gui.py | 63 ++++++++++++++++++++++++++++++++++++++++++---- playlist_engine.py | 33 ++++++++++++++++++------ 2 files changed, 83 insertions(+), 13 deletions(-) diff --git a/main_gui.py b/main_gui.py index dce2ea8..e07282d 100644 --- a/main_gui.py +++ b/main_gui.py @@ -469,6 +469,8 @@ def open_genre_folder(): playlists_dir: str | None = None q: queue.Queue = queue.Queue() cancel_event = threading.Event() + engine_var = getattr(app, "feature_engine_var", tk.StringVar(value="librosa")) + app.feature_engine_var = engine_var def open_path(path: str) -> None: if not path: @@ -488,19 +490,33 @@ def open_instructions(_evt=None): webbrowser.open_new(f"file://{readme}") def dependencies_ok() -> bool: + engine = engine_var.get() or "librosa" missing = [] if playlist_engine.np is None: missing.append("numpy") - if playlist_engine.librosa is None: + if engine == "librosa" and playlist_engine.librosa is None: missing.append("librosa") + if engine == "essentia" and playlist_engine.essentia is None: + missing.append("Essentia") + + essentia_note = ( + "Essentia unavailable; install Essentia to enable the Essentia engine." + if playlist_engine.essentia is None + else "Essentia available." + ) + if missing: dep_status.set( - "Missing dependencies: " + f"Missing dependencies for {engine}: " + ", ".join(missing) - + ". Install with `pip install -r requirements.txt`." + + f". {essentia_note}" ) return False - dep_status.set("Dependencies ready (numpy, librosa)") + + deps = ["numpy", "Essentia" if engine == "essentia" else "librosa"] + dep_status.set( + f"Dependencies ready ({', '.join(deps)}). {essentia_note}" + ) return True def append_log(msg: str) -> None: @@ -544,6 +560,8 @@ def update_controls(): run_btn.config(state="normal" if ready else "disabled") cancel_btn.config(state="normal" if running.get() else "disabled") + engine_var.trace_add("write", lambda *_: update_controls()) + def start_run(): nonlocal total_tracks, playlists_dir if running.get(): @@ -554,6 +572,7 @@ def start_run(): if not dependencies_ok(): messagebox.showerror("Missing Dependencies", dep_status.get()) return + engine = engine_var.get() or "librosa" tracks = gather_tracks(path, getattr(app, "folder_filter", None)) if not tracks: messagebox.showinfo("No Tracks", "No audio files found in the library.") @@ -580,6 +599,7 @@ def worker(): log_callback=lambda m: q.put(("log", m)), progress_callback=lambda c: q.put(("progress", c)), cancel_event=cancel_event, + engine=engine, ) q.put(("done", result)) except Exception as exc: # pragma: no cover - UI surface only @@ -631,6 +651,22 @@ def poll_queue(): ttk.Label(info, textvariable=dep_status, wraplength=360).pack( anchor="w", padx=5, pady=2 ) + engine_frame = ttk.LabelFrame(info, text="Tempo Engine") + engine_frame.pack(fill="x", padx=5, pady=(2, 5)) + ttk.Radiobutton( + engine_frame, + text="Librosa (default)", + variable=engine_var, + value="librosa", + command=update_controls, + ).pack(anchor="w", padx=5, pady=(2, 0)) + ttk.Radiobutton( + engine_frame, + text="Essentia", + variable=engine_var, + value="essentia", + command=update_controls, + ).pack(anchor="w", padx=5, pady=(0, 2)) link = ttk.Label( info, text="View installation instructions", @@ -674,6 +710,17 @@ def poll_queue(): elif name == "Auto-DJ": sel = tk.StringVar() count_var = tk.StringVar(value="20") + engine_var = getattr(app, "feature_engine_var", tk.StringVar(value="librosa")) + app.feature_engine_var = engine_var + + def resolve_engine() -> str: + engine = engine_var.get() + cluster_cfg = getattr(app, "cluster_params", {}) or {} + if not engine: + engine = cluster_cfg.get("feature_engine") or cluster_cfg.get("engine") + if engine in (None, "", "serial", "parallel"): + engine = "librosa" + return engine def open_path(path: str) -> None: if not path: @@ -705,7 +752,10 @@ def generate(): return app.show_log_tab() tracks = gather_tracks(path) - order = autodj_playlist(sel.get(), tracks, n, log_callback=app._log) + engine = resolve_engine() + order = autodj_playlist( + sel.get(), tracks, n, log_callback=app._log, engine=engine + ) outfile = os.path.join(path, "Playlists", "autodj.m3u") write_playlist(order, outfile) messagebox.showinfo("Playlist", f"Written to {outfile}") @@ -977,6 +1027,9 @@ def __init__(self): self.cluster_manager = None self.folder_filter = {"include": [], "exclude": []} + # Shared audio feature/analysis engine selection + self.feature_engine_var = tk.StringVar(value="librosa") + # Cached plugin panels to avoid teardown/rebuild churn self.plugin_views: dict[str, ttk.Frame] = {} self.active_plugin: str | None = None diff --git a/playlist_engine.py b/playlist_engine.py index 23b9359..7277f7e 100644 --- a/playlist_engine.py +++ b/playlist_engine.py @@ -326,10 +326,11 @@ def export_genre_playlists( return {"genres": stats, "playlists_dir": playlists_dir} -def _get_feat(path: str, cache: dict, log_callback): +def _get_feat(path: str, cache: dict, log_callback, engine: TempoEngine = "librosa"): if path not in cache: from clustered_playlists import extract_audio_features - cache[path] = extract_audio_features(path, log_callback) + + cache[path] = extract_audio_features(path, log_callback, engine=engine) return cache[path] @@ -339,32 +340,48 @@ def _dist(a, b) -> float: return math.sqrt(sum((x - y) ** 2 for x, y in zip(a, b))) -def more_like_this(ref_track: str, tracks: list[str], n: int = 10, feature_cache=None, log_callback=None) -> list[str]: +def more_like_this( + ref_track: str, + tracks: list[str], + n: int = 10, + feature_cache=None, + log_callback=None, + engine: TempoEngine = "librosa", +) -> list[str]: if log_callback is None: log_callback = lambda m: None feature_cache = feature_cache or {} - ref_vec = _get_feat(ref_track, feature_cache, log_callback) + ref_vec = _get_feat(ref_track, feature_cache, log_callback, engine) others = [t for t in tracks if t != ref_track] dist = [] for t in others: - vec = _get_feat(t, feature_cache, log_callback) + vec = _get_feat(t, feature_cache, log_callback, engine) d = _dist(ref_vec, vec) dist.append((d, t)) dist.sort() return [t for _d, t in dist[:n]] -def autodj_playlist(start_track: str, tracks: list[str], n: int = 20, feature_cache=None, log_callback=None) -> list[str]: +def autodj_playlist( + start_track: str, + tracks: list[str], + n: int = 20, + feature_cache=None, + log_callback=None, + engine: TempoEngine = "librosa", +) -> list[str]: if log_callback is None: log_callback = lambda m: None feature_cache = feature_cache or {} order = [start_track] remaining = [t for t in tracks if t != start_track] while remaining and len(order) < n: - cur_feat = _get_feat(order[-1], feature_cache, log_callback) + cur_feat = _get_feat(order[-1], feature_cache, log_callback, engine) next_track = min( remaining, - key=lambda t: _dist(cur_feat, _get_feat(t, feature_cache, log_callback)), + key=lambda t: _dist( + cur_feat, _get_feat(t, feature_cache, log_callback, engine) + ), ) order.append(next_track) remaining.remove(next_track) From 54fb4d11d69a589813761e778e0dfe85fbf1eedc Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 14 Dec 2025 19:57:49 -0500 Subject: [PATCH 035/606] Add essentia dependency --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 180b732..14688da 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ ffmpeg-python pyacoustid musicbrainzngs python-vlc -# essentia # disabled on Windows +essentia librosa scipy numpy From debdc3e6b86421156ff582868ab1660defdc07ef Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 14 Dec 2025 20:00:54 -0500 Subject: [PATCH 036/606] Add Essentia optional dependency and docs --- README.md | 20 ++++++++++++++++++++ docs/project_documentation.html | 16 ++++++++++++++++ requirements.txt | 2 +- 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 207e439..d4f7c04 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,26 @@ module. Make sure the Music Indexer package itself is installed (e.g. `pip install .`) so this dependency is available when running the Quality Checker. +### Optional: Essentia audio engine + +Essentia can be used instead of `librosa` for tempo and feature extraction. It +is optional—stick with `librosa` if you don't need it—but enables faster C++ +implementations when available. + +- **Prerequisites** (Linux builds compile C++ code and can take several + minutes): + - Debian/Ubuntu: `sudo apt-get install build-essential libfftw3-dev liblapack-dev libblas-dev libyaml-dev libtag1-dev libchromaprint-dev libsamplerate0-dev libavcodec-dev libavformat-dev libavutil-dev libavresample-dev` + - macOS: `brew install essentia` (installs prebuilt formula with dependencies) + - Windows: no official wheel; use WSL/Linux if you need Essentia. +- **Install** (after prerequisites): + ```bash + pip install essentia==2.1b6 + ``` + +Expect longer build times on Linux the first time you install Essentia. If you +prefer the pure-Python stack, you can continue using `librosa` without this +extra dependency. + ## Quickstart ```bash diff --git a/docs/project_documentation.html b/docs/project_documentation.html index 2d2f307..66cebfb 100644 --- a/docs/project_documentation.html +++ b/docs/project_documentation.html @@ -591,6 +591,22 @@

    2.9 Playlist Engine

  • Wireframe: playlist_wireframe.png
  • +

    2.10 Optional Essentia Audio Engine

    +

    Essentia can replace librosa for tempo analysis and audio feature extraction. It is optional—librosa remains supported—but Essentia offers faster C++ implementations when installed.

    +
      +
    • Prerequisites (Linux builds compile C++ code and may take several minutes): +
        +
      • Debian/Ubuntu: sudo apt-get install build-essential libfftw3-dev liblapack-dev libblas-dev libyaml-dev libtag1-dev libchromaprint-dev libsamplerate0-dev libavcodec-dev libavformat-dev libavutil-dev libavresample-dev
      • +
      • macOS: brew install essentia (installs the formula and dependencies)
      • +
      • Windows: no official wheel; use WSL/Linux if you need Essentia.
      • +
      +
    • +
    • Installation (after prerequisites): +
      pip install essentia==2.1b6
      +
    • +
    • Expect longer build times on Linux the first time you install Essentia. If you prefer the pure-Python stack, keep using librosa without this extra dependency.
    • +
    +

    2.y GUI Front-end (main_gui.py)

    • New menu item Fix Tags.
    • diff --git a/requirements.txt b/requirements.txt index 180b732..7206714 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ ffmpeg-python pyacoustid musicbrainzngs python-vlc -# essentia # disabled on Windows +# essentia==2.1b6 # optional alternative audio engine; see README.md for install steps librosa scipy numpy From f31c66ab0abb95168dccfd14f26c06fce88121f9 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 14 Dec 2025 20:30:39 -0500 Subject: [PATCH 037/606] Add cluster playlist tools to interactive graphs --- cluster_graph_panel.py | 136 +++++++++++++++++++++++++++++++++++++++++ main_gui.py | 87 ++++++++++++++++++++++++++ 2 files changed, 223 insertions(+) diff --git a/cluster_graph_panel.py b/cluster_graph_panel.py index dbefaad..dbfab35 100644 --- a/cluster_graph_panel.py +++ b/cluster_graph_panel.py @@ -1,4 +1,6 @@ import os +import subprocess +import sys import threading from datetime import datetime @@ -67,6 +69,7 @@ def __init__( self.sel_scatter = None self.selected_indices: list[int] = [] self.selected_tracks: list[str] = [] + self.temp_playlist: list[str] = [] # Hover support widgets will be set later via ``setup_hover`` self.hover_panel = None @@ -78,6 +81,13 @@ def __init__( # Preload album art thumbnails for snappy hover updates self.album_thumbnails = [self._load_thumbnail(p) for p in tracks] + # External UI widgets (wired up by main_gui) + self.cluster_combo: ttk.Combobox | None = None + self.cluster_select_var: tk.StringVar | None = None + self.manual_cluster_var: tk.StringVar | None = None + self.temp_listbox: tk.Listbox | None = None + self.temp_status_var: tk.StringVar | None = None + # UI widgets created externally will be assigned after instantiation lasso_btn: ttk.Widget ok_btn: ttk.Button @@ -249,6 +259,37 @@ def create_playlist(self): self.log(f"\u2713 Playlist written: {out}") self.gen_btn.configure(state="disabled") + def create_temp_playlist(self): + """Write the temporary playlist to disk and open the folder.""" + if not self.temp_playlist: + self.log("\u26a0 No songs in temporary playlist") + return + + playlists_dir = os.path.join(self.library_path, "Playlists") + os.makedirs(playlists_dir, exist_ok=True) + ts = datetime.now().strftime("%Y%m%d_%H%M%S") + outfile = os.path.join(playlists_dir, f"ClusterSelection_{ts}.m3u") + try: + write_playlist(self.temp_playlist, outfile) + except Exception as exc: # pragma: no cover - GUI log + self.log(f"\u2717 Failed to write playlist: {exc}") + return + + self.log(f"\u2713 Playlist written: {outfile}") + self._open_folder(playlists_dir) + + def _open_folder(self, path: str) -> None: + """Open ``path`` with the platform default file browser.""" + try: + if sys.platform == "win32": + os.startfile(path) # type: ignore[attr-defined] + elif sys.platform == "darwin": + subprocess.run(["open", path], check=False) + else: + subprocess.run(["xdg-open", path], check=False) + except Exception as exc: # pragma: no cover - OS interaction + messagebox.showerror("Open Folder", f"Could not open {path}: {exc}") + # ─── Helpers ──────────────────────────────────────────────────────────── def _clear_selection(self): self.selected_indices = [] @@ -315,6 +356,7 @@ def _on_clusters_ready(self, labels, params: dict): "[perf] clusters ready (%s) in %.1f ms", self.algo_key, duration_ms ) self._sync_controls() + self._refresh_cluster_options() def recluster(self, params: dict): """Re-run clustering with new ``params`` and redraw the plot.""" @@ -339,6 +381,100 @@ def refresh_control_states(self): """Public hook to sync controls after external widgets are attached.""" self._sync_controls() + def _refresh_cluster_options(self): + """Populate the cluster dropdown with discovered cluster IDs.""" + if not hasattr(self, "labels"): + return + clusters = sorted({int(l) for l in set(self.labels) if l >= 0}) + if self.cluster_combo is not None: + self.cluster_combo.configure(values=[str(c) for c in clusters]) + if self.temp_status_var is not None: + self.temp_status_var.set(f"Clusters available: {len(clusters)}") + + # ─── Cluster-Based Selection Helpers ─────────────────────────────────── + def show_current_playlists(self): + """Display playlists located in the library's Playlists folder.""" + if not self.library_path: + messagebox.showinfo("Playlists", "Select a library first.") + return + + playlists_dir = os.path.join(self.library_path, "Playlists") + if not os.path.isdir(playlists_dir): + messagebox.showinfo("Playlists", "No Playlists folder found yet.") + return + + entries = [] + try: + entries = [ + f + for f in os.listdir(playlists_dir) + if os.path.isfile(os.path.join(playlists_dir, f)) + ] + except OSError as exc: + messagebox.showerror("Playlists", f"Could not read folder: {exc}") + return + + if not entries: + messagebox.showinfo("Playlists", "No playlists found.") + return + + messagebox.showinfo("Playlists", "\n".join(sorted(entries))) + + def _set_temp_playlist(self, tracks: list[str]): + self.temp_playlist = list(dict.fromkeys(tracks)) + self._sync_temp_listbox() + + def _sync_temp_listbox(self): + if self.temp_listbox is None: + return + self.temp_listbox.delete(0, tk.END) + for path in self.temp_playlist: + self.temp_listbox.insert(tk.END, os.path.basename(path)) + if self.temp_status_var is not None: + self.temp_status_var.set(f"Temp playlist tracks: {len(self.temp_playlist)}") + + def load_cluster(self, cluster_id: int, *, replace_temp: bool = True): + """Highlight ``cluster_id`` and optionally seed the temp playlist.""" + if not hasattr(self, "labels"): + messagebox.showinfo("Clusters", "Clusters not ready yet.") + return + + indices = [i for i, lbl in enumerate(self.labels) if lbl == cluster_id] + if not indices: + messagebox.showinfo("Clusters", f"No songs found for cluster {cluster_id}.") + return + + self.selected_indices = indices + self.selected_tracks = [self.tracks[i] for i in indices] + self._update_highlight() + self.log(f"→ Loaded cluster {cluster_id}: {len(indices)} songs") + + if replace_temp: + self._set_temp_playlist(self.selected_tracks) + + def add_highlight_to_temp(self): + """Append currently highlighted songs to the temp playlist.""" + if not self.selected_indices: + messagebox.showinfo("Selection", "No highlighted songs to add.") + return + current = list(self.temp_playlist) + for idx in self.selected_indices: + track = self.tracks[idx] + if track not in current: + current.append(track) + self._set_temp_playlist(current) + + def remove_selected_from_temp(self): + """Remove highlighted entries from the temp playlist listbox.""" + if self.temp_listbox is None: + return + selected = list(self.temp_listbox.curselection()) + if not selected: + return + remaining = [t for i, t in enumerate(self.temp_playlist) if i not in selected] + self._set_temp_playlist(remaining) + + def open_param_dialog(self): """Show dialog to edit HDBSCAN parameters and redraw clusters.""" dlg = tk.Toplevel(self) diff --git a/main_gui.py b/main_gui.py index e07282d..e3078ad 100644 --- a/main_gui.py +++ b/main_gui.py @@ -789,6 +789,7 @@ def generate(): container.pack(fill="both", expand=True) container.rowconfigure(0, weight=1) container.columnconfigure(0, weight=1) + container.columnconfigure(1, weight=0) if cluster_manager is None: return frame @@ -809,6 +810,91 @@ def generate(): ) panel.grid(row=0, column=0, sticky="nsew", pady=(0, 5)) + side_tools = ttk.Frame(container) + side_tools.grid(row=0, column=1, rowspan=3, sticky="ns", padx=(10, 0)) + side_tools.columnconfigure(0, weight=1) + side_tools.rowconfigure(2, weight=1) + + playlist_btn = ttk.Button( + side_tools, text="Current Playlists", command=panel.show_current_playlists + ) + playlist_btn.grid(row=0, column=0, sticky="ew", pady=(0, 10)) + + cluster_box = ttk.LabelFrame(side_tools, text="Cluster Loader") + cluster_box.grid(row=1, column=0, sticky="nsew", pady=(0, 10)) + cluster_box.columnconfigure(1, weight=1) + + ttk.Label(cluster_box, text="Pick cluster:").grid( + row=0, column=0, sticky="w", padx=5, pady=(5, 2) + ) + panel.cluster_select_var = tk.StringVar() + panel.cluster_combo = ttk.Combobox( + cluster_box, + textvariable=panel.cluster_select_var, + state="readonly", + width=10, + ) + panel.cluster_combo.grid(row=0, column=1, sticky="ew", padx=5, pady=(5, 2)) + + ttk.Label(cluster_box, text="Or enter #:").grid( + row=1, column=0, sticky="w", padx=5, pady=(0, 2) + ) + panel.manual_cluster_var = tk.StringVar() + manual_entry = ttk.Entry( + cluster_box, textvariable=panel.manual_cluster_var, width=10 + ) + manual_entry.grid(row=1, column=1, sticky="ew", padx=5, pady=(0, 2)) + + def _load_cluster(): + choice = panel.manual_cluster_var.get().strip() or panel.cluster_select_var.get().strip() + if not choice: + messagebox.showinfo("Clusters", "Choose a cluster first.") + return + try: + cid = int(choice) + except ValueError: + messagebox.showinfo("Clusters", f"{choice} is not a valid number") + return + panel.load_cluster(cid) + + ttk.Button(cluster_box, text="Load Cluster", command=_load_cluster).grid( + row=2, column=0, columnspan=2, sticky="ew", padx=5, pady=(0, 5) + ) + + panel.temp_status_var = tk.StringVar(value="Clusters available: 0") + ttk.Label(cluster_box, textvariable=panel.temp_status_var).grid( + row=3, column=0, columnspan=2, sticky="w", padx=5, pady=(0, 5) + ) + + temp_box = ttk.LabelFrame(side_tools, text="Temporary Playlist") + temp_box.grid(row=2, column=0, sticky="nsew") + temp_box.columnconfigure(0, weight=1) + temp_box.rowconfigure(0, weight=1) + + list_frame = ttk.Frame(temp_box) + list_frame.grid(row=0, column=0, sticky="nsew", padx=5, pady=5) + list_frame.columnconfigure(0, weight=1) + + temp_scroll = ttk.Scrollbar(list_frame, orient="vertical") + panel.temp_listbox = tk.Listbox( + list_frame, height=12, selectmode="extended", yscrollcommand=temp_scroll.set + ) + panel.temp_listbox.pack(side="left", fill="both", expand=True) + temp_scroll.config(command=panel.temp_listbox.yview) + temp_scroll.pack(side="right", fill="y") + + ttk.Button( + temp_box, text="Add Highlighted Songs", command=panel.add_highlight_to_temp + ).grid(row=1, column=0, sticky="ew", padx=5, pady=(0, 5)) + + ttk.Button( + temp_box, text="Remove Selected", command=panel.remove_selected_from_temp + ).grid(row=2, column=0, sticky="ew", padx=5, pady=(0, 5)) + + ttk.Button(temp_box, text="Create Playlist", command=panel.create_temp_playlist).grid( + row=3, column=0, sticky="ew", padx=5, pady=(0, 5) + ) + ttk.Separator(container, orient="horizontal").grid(row=1, column=0, sticky="ew") btn_frame = ttk.Frame(container) @@ -881,6 +967,7 @@ def _auto_create_all(): hover_panel.place_forget() panel.setup_hover(hover_panel, art_lbl, title_lbl, artist_lbl) + panel._refresh_cluster_options() panel.refresh_control_states() return frame From 541baf26f12955b42c2f428a4ab9e40f3149fc4e Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 14 Dec 2025 20:42:09 -0500 Subject: [PATCH 038/606] Improve clustered playlist robustness and logging --- cluster_graph_panel.py | 2 ++ clustered_playlists.py | 52 +++++++++++++++++++++++++++++++++++++++--- main_gui.py | 37 ++++++++++++++++++++++++++---- 3 files changed, 84 insertions(+), 7 deletions(-) diff --git a/cluster_graph_panel.py b/cluster_graph_panel.py index dbfab35..01b6242 100644 --- a/cluster_graph_panel.py +++ b/cluster_graph_panel.py @@ -15,6 +15,7 @@ from matplotlib.path import Path from playlist_generator import write_playlist +from clustered_playlists import log_cluster_summary from io import BytesIO from PIL import Image, ImageTk @@ -351,6 +352,7 @@ def _on_clusters_ready(self, labels, params: dict): if self._loading_lbl.winfo_exists(): self._loading_lbl.pack_forget() self._draw_clusters(labels) + log_cluster_summary(labels, self.log) if duration_ms is not None: logger.info( "[perf] clusters ready (%s) in %.1f ms", self.algo_key, duration_ms diff --git a/clustered_playlists.py b/clustered_playlists.py index 7d74bca..4f8a482 100644 --- a/clustered_playlists.py +++ b/clustered_playlists.py @@ -1,5 +1,6 @@ import importlib.util import os +from collections import Counter from concurrent.futures import ProcessPoolExecutor, as_completed import multiprocessing from typing import Literal @@ -194,7 +195,13 @@ def cluster_tracks( log_callback = lambda msg: None if method == "kmeans": - n_clusters = int(kwargs.get("n_clusters", 5)) + n_clusters = max(1, int(kwargs.get("n_clusters", 5))) + if n_clusters > len(feature_matrix): + log_callback( + f"→ Requested {n_clusters} clusters but only " + f"{len(feature_matrix)} tracks; reducing to {len(feature_matrix)}" + ) + n_clusters = len(feature_matrix) log_callback( f"⚙ Clustering {len(feature_matrix)} tracks into {n_clusters} groups …" ) @@ -202,10 +209,23 @@ def cluster_tracks( feature_matrix ) else: - min_cluster_size = int(kwargs.get("min_cluster_size", 5)) + min_cluster_size = max(2, int(kwargs.get("min_cluster_size", 5))) + if min_cluster_size > len(feature_matrix): + log_callback( + f"→ Min cluster size {min_cluster_size} exceeds track count " + f"{len(feature_matrix)}; reducing to {len(feature_matrix)}" + ) + min_cluster_size = len(feature_matrix) extra: dict = {} if "min_samples" in kwargs: - extra["min_samples"] = int(kwargs["min_samples"]) + min_samples = max(1, int(kwargs["min_samples"])) + if min_samples > len(feature_matrix): + log_callback( + f"→ Min samples {min_samples} exceeds track count " + f"{len(feature_matrix)}; reducing to {len(feature_matrix)}" + ) + min_samples = len(feature_matrix) + extra["min_samples"] = min_samples if "cluster_selection_epsilon" in kwargs: extra["cluster_selection_epsilon"] = float( kwargs["cluster_selection_epsilon"] @@ -221,6 +241,31 @@ def cluster_tracks( return labels +def log_cluster_summary(labels: "np.ndarray", log_callback) -> None: + """Log a compact summary of cluster counts and noise.""" + + if log_callback is None: + return + + total = len(labels) + noise = sum(1 for l in labels if l < 0) + clusters = Counter(int(l) for l in labels if l >= 0) + if not clusters: + log_callback( + f"→ All {total} tracks were marked as noise; try adjusting parameters" + ) + return + + cluster_count = len(clusters) + log_callback( + f"→ Cluster summary: {cluster_count} clusters, {noise} noise tracks" + ) + top_sizes = ", ".join( + f"{cid}: {size}" for cid, size in clusters.most_common(5) + ) + log_callback(f"→ Largest clusters (id: size): {top_sizes}") + + def generate_clustered_playlists( tracks, root_path: str, @@ -291,6 +336,7 @@ def generate_clustered_playlists( log_callback( f"✓ Clustering complete: found {len(set([l for l in labels if l >= 0]) )} clusters" ) + log_cluster_summary(labels, log_callback) playlists_dir = os.path.join(root_path, "Playlists") os.makedirs(playlists_dir, exist_ok=True) diff --git a/main_gui.py b/main_gui.py index e3078ad..393e9b8 100644 --- a/main_gui.py +++ b/main_gui.py @@ -86,6 +86,8 @@ FilterFn = Callable[[FileRecord], bool] _cached_filters = None +logger = logging.getLogger(__name__) + def make_filters(ex_no_diff: bool, ex_skipped: bool, show_all: bool) -> List[FilterFn]: global _cached_filters @@ -146,7 +148,15 @@ def create_panel_for_plugin(app, name: str, parent: tk.Widget) -> ttk.Frame: from sklearn.cluster import KMeans def km_func(X, p): - return KMeans(n_clusters=p["n_clusters"]).fit_predict(X) + requested = max(1, int(p["n_clusters"])) + effective = min(requested, len(X)) + if effective != requested: + logger.info( + "Adjusting kmeans clusters from %s to %s due to dataset size", + requested, + effective, + ) + return KMeans(n_clusters=effective).fit_predict(X) n_clusters = 5 if cluster_cfg and cluster_cfg.get("method") == "kmeans": @@ -160,11 +170,30 @@ def km_func(X, p): from hdbscan import HDBSCAN def km_func(X, p): - kwargs = {"min_cluster_size": p["min_cluster_size"]} + min_cluster_size = max(2, int(p["min_cluster_size"])) + if min_cluster_size > len(X): + logger.info( + "Adjusting min_cluster_size from %s to %s due to dataset size", + min_cluster_size, + len(X), + ) + min_cluster_size = len(X) + + kwargs = {"min_cluster_size": min_cluster_size} if "min_samples" in p: - kwargs["min_samples"] = p["min_samples"] + min_samples = max(1, int(p["min_samples"])) + if min_samples > len(X): + logger.info( + "Adjusting min_samples from %s to %s due to dataset size", + min_samples, + len(X), + ) + min_samples = len(X) + kwargs["min_samples"] = min_samples if "cluster_selection_epsilon" in p: - kwargs["cluster_selection_epsilon"] = p["cluster_selection_epsilon"] + kwargs["cluster_selection_epsilon"] = float( + p["cluster_selection_epsilon"] + ) return HDBSCAN(**kwargs).fit_predict(X) min_cs = 5 From 4ad2fec181a15e1be0d15d93c8b68defb8b68e79 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 14 Dec 2025 20:53:08 -0500 Subject: [PATCH 039/606] Disable auto clustered playlists and add show-all highlight --- cluster_graph_panel.py | 20 ++++++++++++++++++++ clustered_playlists.py | 21 +++------------------ main_gui.py | 10 +++++++--- 3 files changed, 30 insertions(+), 21 deletions(-) diff --git a/cluster_graph_panel.py b/cluster_graph_panel.py index 01b6242..585119b 100644 --- a/cluster_graph_panel.py +++ b/cluster_graph_panel.py @@ -454,6 +454,26 @@ def load_cluster(self, cluster_id: int, *, replace_temp: bool = True): if replace_temp: self._set_temp_playlist(self.selected_tracks) + def highlight_temp_playlist(self): + """Highlight all points currently stored in the temp playlist.""" + if not self.temp_playlist: + messagebox.showinfo("Selection", "No songs in the temporary playlist.") + return + + temp_set = set(self.temp_playlist) + indices = [i for i, track in enumerate(self.tracks) if track in temp_set] + if not indices: + messagebox.showinfo( + "Selection", + "No matching songs from the temporary playlist were found in this view.", + ) + return + + self.selected_indices = indices + self.selected_tracks = [self.tracks[i] for i in indices] + self._update_highlight() + self.log(f"→ Highlighted {len(indices)} songs from the temporary playlist") + def add_highlight_to_temp(self): """Append currently highlighted songs to the temp playlist.""" if not self.selected_indices: diff --git a/clustered_playlists.py b/clustered_playlists.py index 4f8a482..6dfa5cd 100644 --- a/clustered_playlists.py +++ b/clustered_playlists.py @@ -275,7 +275,7 @@ def generate_clustered_playlists( engine: str = "serial", feature_engine: AudioFeatureEngine = "librosa", ) -> None: - """Create clustered playlists for the given tracks.""" + """Create clustered data for the given tracks without writing playlists.""" if log_callback is None: log_callback = lambda msg: None @@ -338,21 +338,6 @@ def generate_clustered_playlists( ) log_cluster_summary(labels, log_callback) - playlists_dir = os.path.join(root_path, "Playlists") - os.makedirs(playlists_dir, exist_ok=True) - - for cluster_id in sorted(set(labels)): - if cluster_id < 0: - continue - playlist = [tracks[i] for i, lbl in enumerate(labels) if lbl == cluster_id] - outfile = os.path.join(playlists_dir, f"{method}_cluster_{cluster_id}.m3u") - try: - with open(outfile, "w", encoding="utf-8") as pf: - for p in playlist: - pf.write(os.path.relpath(p, playlists_dir) + "\n") - log_callback(f"→ Writing clustered playlist: {outfile}") - except Exception as e: - log_callback(f"\u2717 Failed to write {outfile}: {e}") - - log_callback("✓ Clustered playlist generation finished") + log_callback("→ Automatic playlist export disabled; manage selections manually.") + return feats diff --git a/main_gui.py b/main_gui.py index 393e9b8..b9da5cd 100644 --- a/main_gui.py +++ b/main_gui.py @@ -912,16 +912,20 @@ def _load_cluster(): temp_scroll.config(command=panel.temp_listbox.yview) temp_scroll.pack(side="right", fill="y") + ttk.Button(temp_box, text="Show All", command=panel.highlight_temp_playlist).grid( + row=1, column=0, sticky="ew", padx=5, pady=(0, 5) + ) + ttk.Button( temp_box, text="Add Highlighted Songs", command=panel.add_highlight_to_temp - ).grid(row=1, column=0, sticky="ew", padx=5, pady=(0, 5)) + ).grid(row=2, column=0, sticky="ew", padx=5, pady=(0, 5)) ttk.Button( temp_box, text="Remove Selected", command=panel.remove_selected_from_temp - ).grid(row=2, column=0, sticky="ew", padx=5, pady=(0, 5)) + ).grid(row=3, column=0, sticky="ew", padx=5, pady=(0, 5)) ttk.Button(temp_box, text="Create Playlist", command=panel.create_temp_playlist).grid( - row=3, column=0, sticky="ew", padx=5, pady=(0, 5) + row=4, column=0, sticky="ew", padx=5, pady=(0, 5) ) ttk.Separator(container, orient="horizontal").grid(row=1, column=0, sticky="ew") From d60433df19010f2ac2e9437453187b555ef034f2 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 14 Dec 2025 21:09:57 -0500 Subject: [PATCH 040/606] Add lasso-driven removal for temp playlists --- cluster_graph_panel.py | 99 ++++++++++++++++++++++++++++++++++++++---- main_gui.py | 5 ++- 2 files changed, 93 insertions(+), 11 deletions(-) diff --git a/cluster_graph_panel.py b/cluster_graph_panel.py index 585119b..82d61f1 100644 --- a/cluster_graph_panel.py +++ b/cluster_graph_panel.py @@ -71,6 +71,9 @@ def __init__( self.selected_indices: list[int] = [] self.selected_tracks: list[str] = [] self.temp_playlist: list[str] = [] + self._remove_lasso_started = False + self._remove_lasso_prev_state = False + self._remove_lasso_confirm_pending = False # Hover support widgets will be set later via ``setup_hover`` self.hover_panel = None @@ -88,6 +91,9 @@ def __init__( self.manual_cluster_var: tk.StringVar | None = None self.temp_listbox: tk.Listbox | None = None self.temp_status_var: tk.StringVar | None = None + self.temp_remove_btn: ttk.Button | None = None + + self.canvas.mpl_connect("button_press_event", self._on_canvas_click) # UI widgets created externally will be assigned after instantiation lasso_btn: ttk.Widget @@ -131,16 +137,27 @@ def _worker(): # ─── Lasso Handling ───────────────────────────────────────────────────── def toggle_lasso(self): """Enter or exit lasso selection mode.""" - if self.lasso is None: - self._clear_selection() - self.lasso = LassoSelector(self.ax, onselect=self._on_lasso_select) + self._cancel_temp_lasso_removal() + self._set_lasso_enabled(self.lasso is None) + + def _set_lasso_enabled(self, active: bool): + """Toggle matplotlib lasso state and sync UI controls.""" + + self._clear_selection() + if active: + if self.lasso is None: + self.lasso = LassoSelector(self.ax, onselect=self._on_lasso_select) self.log("Lasso mode enabled – draw a shape") else: - self.lasso.disconnect_events() - self.lasso = None - self.log("Lasso mode disabled") - self._clear_selection() + if self.lasso is not None: + self.lasso.disconnect_events() + self.lasso = None + self.log("Lasso mode disabled") + else: + return + if hasattr(self, "lasso_var"): + self.lasso_var.set(active) self.ok_btn.configure(state="disabled") self.gen_btn.configure(state="disabled") self.canvas.draw_idle() @@ -221,6 +238,14 @@ def _on_lasso_select(self, verts): self.selected_indices = [i for i, v in enumerate(mask) if v] self.log(f"\u2192 {len(self.selected_indices)} songs selected") self._update_highlight() + if self._remove_lasso_started: + self._remove_lasso_confirm_pending = bool(self.selected_indices) + if self.temp_remove_btn is not None: + text = "Press to Confirm" if self.selected_indices else "Lasso Selection" + self.temp_remove_btn.configure(text=text) + if not self.selected_indices: + self.ok_btn.configure(state="disabled") + return if self.selected_indices: self.ok_btn.configure(state="normal") else: @@ -299,6 +324,10 @@ def _clear_selection(self): self.sel_scatter.remove() self.sel_scatter = None + def _on_canvas_click(self, _event) -> None: + if self._remove_lasso_started and self._remove_lasso_confirm_pending: + self._cancel_temp_lasso_removal() + def _update_highlight(self): if self.sel_scatter is not None: self.sel_scatter.remove() @@ -490,11 +519,63 @@ def remove_selected_from_temp(self): """Remove highlighted entries from the temp playlist listbox.""" if self.temp_listbox is None: return + + if self._remove_lasso_confirm_pending: + self._confirm_temp_lasso_removal() + return + selected = list(self.temp_listbox.curselection()) - if not selected: + if selected: + self._cancel_temp_lasso_removal() + remaining = [t for i, t in enumerate(self.temp_playlist) if i not in selected] + self._set_temp_playlist(remaining) return - remaining = [t for i, t in enumerate(self.temp_playlist) if i not in selected] + + self._begin_temp_lasso_removal() + + def _begin_temp_lasso_removal(self) -> None: + """Switch to lasso mode to pick songs for removal.""" + + self._remove_lasso_started = True + self._remove_lasso_prev_state = self.lasso is not None + self._remove_lasso_confirm_pending = False + if self.temp_remove_btn is not None: + self.temp_remove_btn.configure(text="Lasso Selection") + self._set_lasso_enabled(True) + self.log("Use lasso to select songs to remove from the playlist") + + def _confirm_temp_lasso_removal(self) -> None: + """Remove any lasso-selected points that exist in the temp playlist.""" + + if not self.selected_indices: + self.log("⚠ No points selected; removal cancelled") + self._cancel_temp_lasso_removal() + return + + selected_tracks = [self.tracks[i] for i in self.selected_indices] + temp_set = set(self.temp_playlist) + to_remove = [t for t in selected_tracks if t in temp_set] + if not to_remove: + self.log("⚠ None of the lassoed songs are in the temporary playlist") + self._cancel_temp_lasso_removal() + return + + remaining = [t for t in self.temp_playlist if t not in set(to_remove)] self._set_temp_playlist(remaining) + self.log(f"→ Removed {len(to_remove)} songs from the temporary playlist") + self._cancel_temp_lasso_removal() + + def _cancel_temp_lasso_removal(self) -> None: + """Reset state for lasso-driven removal.""" + + if self._remove_lasso_started and not self._remove_lasso_prev_state: + self._set_lasso_enabled(False) + self._remove_lasso_started = False + self._remove_lasso_prev_state = False + self._remove_lasso_confirm_pending = False + if self.temp_remove_btn is not None: + self.temp_remove_btn.configure(text="Remove Selected") + self._clear_selection() def open_param_dialog(self): diff --git a/main_gui.py b/main_gui.py index b9da5cd..53e754a 100644 --- a/main_gui.py +++ b/main_gui.py @@ -920,9 +920,10 @@ def _load_cluster(): temp_box, text="Add Highlighted Songs", command=panel.add_highlight_to_temp ).grid(row=2, column=0, sticky="ew", padx=5, pady=(0, 5)) - ttk.Button( + panel.temp_remove_btn = ttk.Button( temp_box, text="Remove Selected", command=panel.remove_selected_from_temp - ).grid(row=3, column=0, sticky="ew", padx=5, pady=(0, 5)) + ) + panel.temp_remove_btn.grid(row=3, column=0, sticky="ew", padx=5, pady=(0, 5)) ttk.Button(temp_box, text="Create Playlist", command=panel.create_temp_playlist).grid( row=4, column=0, sticky="ew", padx=5, pady=(0, 5) From 9b356fc863243829216c5c7fdbecc1859d29de5d Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 14 Dec 2025 21:16:41 -0500 Subject: [PATCH 041/606] Prompt for playlist filename before saving --- cluster_graph_panel.py | 52 ++++++++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/cluster_graph_panel.py b/cluster_graph_panel.py index 82d61f1..77e8a72 100644 --- a/cluster_graph_panel.py +++ b/cluster_graph_panel.py @@ -8,7 +8,7 @@ import time import logging import tkinter as tk -from tkinter import ttk, messagebox +from tkinter import ttk, messagebox, filedialog from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg from matplotlib.figure import Figure from matplotlib.widgets import LassoSelector @@ -267,22 +267,18 @@ def create_playlist(self): self.log("\u26a0 No songs selected") return - playlists_dir = os.path.join(self.library_path, "Playlists") - os.makedirs(playlists_dir, exist_ok=True) - ts = datetime.now().strftime("%Y%m%d_%H%M%S") - base = os.path.join(playlists_dir, f"CustomLasso_{ts}.m3u") - out = base - idx = 1 - while os.path.exists(out): - out = base.replace(".m3u", f"_{idx}.m3u") - idx += 1 + outfile = self._prompt_playlist_destination("CustomLasso") + if not outfile: + self.log("\u26a0 Playlist save cancelled") + return try: - write_playlist(self.selected_tracks, out) + write_playlist(self.selected_tracks, outfile) except Exception as e: # pragma: no cover - simple GUI log self.log(f"\u2717 Failed to write playlist: {e}") else: - self.log(f"\u2713 Playlist written: {out}") + self.log(f"\u2713 Playlist written: {outfile}") + self._open_folder(os.path.dirname(outfile)) self.gen_btn.configure(state="disabled") def create_temp_playlist(self): @@ -291,10 +287,10 @@ def create_temp_playlist(self): self.log("\u26a0 No songs in temporary playlist") return - playlists_dir = os.path.join(self.library_path, "Playlists") - os.makedirs(playlists_dir, exist_ok=True) - ts = datetime.now().strftime("%Y%m%d_%H%M%S") - outfile = os.path.join(playlists_dir, f"ClusterSelection_{ts}.m3u") + outfile = self._prompt_playlist_destination("ClusterSelection") + if not outfile: + self.log("\u26a0 Playlist save cancelled") + return try: write_playlist(self.temp_playlist, outfile) except Exception as exc: # pragma: no cover - GUI log @@ -302,7 +298,29 @@ def create_temp_playlist(self): return self.log(f"\u2713 Playlist written: {outfile}") - self._open_folder(playlists_dir) + self._open_folder(os.path.dirname(outfile)) + + def _prompt_playlist_destination(self, default_prefix: str) -> str | None: + """Open a save dialog and return the chosen playlist path in Playlists.""" + + playlists_dir = os.path.join(self.library_path, "Playlists") + os.makedirs(playlists_dir, exist_ok=True) + ts = datetime.now().strftime("%Y%m%d_%H%M%S") + default_name = f"{default_prefix}_{ts}.m3u" + + chosen = filedialog.asksaveasfilename( + parent=self, + title="Save Playlist As", + defaultextension=".m3u", + initialdir=playlists_dir, + initialfile=default_name, + filetypes=[("M3U Playlist", "*.m3u"), ("All Files", "*.*")], + ) + + if not chosen: + return None + + return os.path.join(playlists_dir, os.path.basename(chosen)) def _open_folder(self, path: str) -> None: """Open ``path`` with the platform default file browser.""" From 91733171726be81a6d1b64fa90e887610818e4cf Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 14 Dec 2025 21:49:38 -0500 Subject: [PATCH 042/606] Simplify VLC preview start behavior --- gui/audio_preview.py | 47 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/gui/audio_preview.py b/gui/audio_preview.py index c81d97f..1107cca 100644 --- a/gui/audio_preview.py +++ b/gui/audio_preview.py @@ -1,6 +1,7 @@ import logging import os import threading +import time from typing import Optional @@ -29,6 +30,7 @@ def __init__(self, on_done=None) -> None: self._player: Optional["vlc.MediaPlayer"] = None self._event_mgr = None self._is_playing = False + self._stop_timer: threading.Timer | None = None self._init_error: Exception | None = VLC_IMPORT_ERROR self._setup_player() @@ -69,6 +71,9 @@ def stop_preview(self) -> None: with self._play_lock: if not self._player: return + if self._stop_timer: + self._stop_timer.cancel() + self._stop_timer = None try: self._logger.debug("Stopping VLC preview playback") self._player.stop() @@ -98,7 +103,18 @@ def play_clip(self, path: str, start_ms: int = 30000, duration_ms: int = 15000) ) self._player.set_media(media) try: + # Start muted to avoid leaking audio before the preview offset. + self._player.audio_set_mute(True) self._player.play() + threading.Thread( + target=self._unmute_when_ready, + daemon=True, + ).start() + if duration_ms > 0: + self._stop_timer = threading.Timer( + duration_ms / 1000, self.stop_preview + ) + self._stop_timer.start() except Exception as exc: self._is_playing = False self._logger.exception("Failed to start VLC playback") @@ -107,6 +123,8 @@ def play_clip(self, path: str, start_ms: int = 30000, duration_ms: int = 15000) def _handle_end(self, event=None) -> None: # pragma: no cover - event driven with self._play_lock: self._is_playing = False + if self._stop_timer: + self._stop_timer = None self._logger.debug("VLC preview finished") if self._on_done: try: @@ -117,9 +135,38 @@ def _handle_end(self, event=None) -> None: # pragma: no cover - event driven def _handle_error(self, event=None) -> None: # pragma: no cover - event driven with self._play_lock: self._is_playing = False + if self._stop_timer: + self._stop_timer = None self._logger.error("VLC reported an error during preview playback") if self._on_done: try: self._on_done() except Exception: self._logger.exception("Error in preview error callback") + + def _unmute_when_ready(self) -> None: + """Unmute once VLC has reached a stable playback state.""" + + if not self._player: + return + + for _ in range(40): + with self._play_lock: + if not self._is_playing: + return + state = self._player.get_state() + if state in (vlc.State.Playing, vlc.State.Paused): + break + time.sleep(0.025) + + # Give VLC a brief moment to settle at the start-time offset. + time.sleep(0.05) + + with self._play_lock: + if not self._is_playing: + return + + try: + self._player.audio_set_mute(False) + except Exception: + self._logger.exception("Failed to unmute VLC preview") From e80a7e2f373f7ed8a6b5c476c01c14607606af12 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 14 Dec 2025 22:15:18 -0500 Subject: [PATCH 043/606] Show log tab when starting clustering --- main_gui.py | 1 + 1 file changed, 1 insertion(+) diff --git a/main_gui.py b/main_gui.py index 53e754a..20a7a20 100644 --- a/main_gui.py +++ b/main_gui.py @@ -2411,6 +2411,7 @@ def _start_cluster_playlists(self, method: str, params: dict, engine: str, dlg): path = self.require_library() if not path: return + self.show_log_tab() threading.Thread( target=self._run_cluster_generation, args=(path, method, params, engine), From 0835a9899befa2400e03fc1e3c21049059b63106 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 14 Dec 2025 22:27:44 -0500 Subject: [PATCH 044/606] Embed genre normalizer in genre sort panel --- main_gui.py | 60 +++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 9 deletions(-) diff --git a/main_gui.py b/main_gui.py index 20a7a20..86de1ee 100644 --- a/main_gui.py +++ b/main_gui.py @@ -229,6 +229,8 @@ def km_func(X, p): genre_vars: dict[str, tk.BooleanVar] = {} current_buckets: dict[str, list[str]] = {} planned_paths: dict[str, str] = {} + norm_status = tk.StringVar(value="Select a library to enable normalization.") + norm_btn: ttk.Button | None = None def append_log(msg: str) -> None: log_box.configure(state="normal") @@ -299,6 +301,15 @@ def update_controls(): ) select_all_btn.config(state="normal" if has_results and ready else "disabled") select_none_btn.config(state="normal" if has_results and ready else "disabled") + norm_status.set( + os.path.join(app.library_path, ".genre_mapping.json") + if app.library_path + else "Select a library to enable normalization." + ) + if norm_btn: + norm_btn.config( + state="normal" if app.library_path and not running.get() else "disabled" + ) def open_path(path: str) -> None: if not path: @@ -434,7 +445,15 @@ def open_genre_folder(): os.makedirs(path, exist_ok=True) open_path(path) - info = ttk.Frame(frame) + paned = ttk.Panedwindow(frame, orient="horizontal") + paned.pack(fill="both", expand=True) + + main_area = ttk.Frame(paned) + side_panel = ttk.Frame(paned, width=280) + paned.add(main_area, weight=3) + paned.add(side_panel, weight=2) + + info = ttk.Frame(main_area) info.pack(fill="x", pady=(0, 6)) ttk.Label(info, textvariable=lib_var).pack(side="left") open_btn = ttk.Button(info, text="Open Genre Playlists", command=open_genre_folder) @@ -444,13 +463,13 @@ def open_genre_folder(): cancel_btn = ttk.Button(info, text="Cancel", command=cancel_run, state="disabled") cancel_btn.pack(side="right") - prog_frame = ttk.Frame(frame) + prog_frame = ttk.Frame(main_area) prog_frame.pack(fill="x", pady=(0, 6)) progress = ttk.Progressbar(prog_frame, mode="determinate") progress.pack(fill="x") ttk.Label(prog_frame, textvariable=progress_var).pack(anchor="w") - selection_bar = ttk.Frame(frame) + selection_bar = ttk.Frame(main_area) selection_bar.pack(fill="x", pady=(0, 4)) select_all_btn = ttk.Button( selection_bar, text="Select All", command=select_all, state="disabled" @@ -468,7 +487,7 @@ def open_genre_folder(): ) export_btn.pack(side="right") - stats_container = ttk.Frame(frame) + stats_container = ttk.Frame(main_area) stats_container.pack(fill="both", pady=(0, 6), expand=True) stats_canvas = tk.Canvas(stats_container, height=220) stats_scroll = ttk.Scrollbar(stats_container, orient="vertical", command=stats_canvas.yview) @@ -483,10 +502,36 @@ def open_genre_folder(): stats_scroll.pack(side="right", fill="y") render_stats(None) - ttk.Label(frame, text="Activity Log:").pack(anchor="w") - log_box = ScrolledText(frame, height=10, wrap="word", state="disabled") + ttk.Label(main_area, text="Activity Log:").pack(anchor="w") + log_box = ScrolledText(main_area, height=10, wrap="word", state="disabled") log_box.pack(fill="both", expand=True) + # ── Genre Normalizer Side Panel ───────────────────────────────── + normalizer_box = ttk.LabelFrame(side_panel, text="Genre Normalizer") + normalizer_box.pack(fill="both", expand=True, padx=(10, 0)) + + ttk.Label( + normalizer_box, + text=( + "Create or edit a mapping file before exporting playlists. " + "The mapping will normalize genres during sorting." + ), + wraplength=240, + justify="left", + ).pack(fill="x", padx=8, pady=(8, 6)) + + ttk.Label(normalizer_box, text="Mapping file:").pack(anchor="w", padx=8) + ttk.Label(normalizer_box, textvariable=norm_status, wraplength=240).pack( + fill="x", padx=8 + ) + + norm_btn = ttk.Button( + normalizer_box, + text="Open Genre Normalizer", + command=lambda: app._open_genre_normalizer(), + ) + norm_btn.pack(pady=8, padx=8, anchor="e") + update_controls() elif name == "Tempo/Energy Buckets": @@ -1269,9 +1314,6 @@ def build_ui(self): command=lambda: list_unique_genres(self.require_library()), ) tools_menu.add_separator() - tools_menu.add_command( - label="Genre Normalizer", command=self._open_genre_normalizer - ) tools_menu.add_command(label="Reset Tag-Fix Log", command=self.reset_tagfix_log) menubar.add_cascade(label="Tools", menu=tools_menu) From c82eafcc9b7c6e833000916ba8ca1ff58b873ede Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 14 Dec 2025 22:34:00 -0500 Subject: [PATCH 045/606] Add playlist builder to player tab --- main_gui.py | 190 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 190 insertions(+) diff --git a/main_gui.py b/main_gui.py index 86de1ee..b767ac9 100644 --- a/main_gui.py +++ b/main_gui.py @@ -32,6 +32,7 @@ from tkinter.scrolledtext import ScrolledText import textwrap import time +from datetime import datetime from validator import validate_soundvault_structure from music_indexer_api import ( @@ -82,6 +83,7 @@ import playlist_engine from playlist_engine import bucket_by_tempo_energy, autodj_playlist from controllers.cluster_controller import gather_tracks +from playlist_generator import write_playlist FilterFn = Callable[[FileRecord], bool] _cached_filters = None @@ -1259,6 +1261,11 @@ def __init__(self): self._player_load_thread: threading.Thread | None = None self.player_art_image: ImageTk.PhotoImage | None = None self.player_search_var = tk.StringVar(value="") + self.player_temp_playlist: list[str] = [] + self.player_playlist_status_var = tk.StringVar( + value="No songs in the playlist builder yet." + ) + self.player_playlist_listbox: tk.Listbox | None = None # assume ffmpeg is available without performing checks self.ffmpeg_available = True @@ -1731,6 +1738,51 @@ def build_ui(self): self.player_art_caption.pack(fill="x") self._update_player_art(None) + playlist_box = ttk.LabelFrame(art_panel, text="Playlist Builder") + playlist_box.pack(fill="both", expand=True, pady=(10, 0)) + ttk.Label( + playlist_box, textvariable=self.player_playlist_status_var + ).pack(anchor="w", padx=5, pady=(5, 2)) + + pl_list_frame = ttk.Frame(playlist_box) + pl_list_frame.pack(fill="both", expand=True, padx=5) + pl_scroll = ttk.Scrollbar(pl_list_frame, orient="vertical") + self.player_playlist_listbox = tk.Listbox( + pl_list_frame, + selectmode="extended", + height=10, + yscrollcommand=pl_scroll.set, + ) + self.player_playlist_listbox.pack(side="left", fill="both", expand=True) + pl_scroll.config(command=self.player_playlist_listbox.yview) + pl_scroll.pack(side="right", fill="y") + + btn_row1 = ttk.Frame(playlist_box) + btn_row1.pack(fill="x", padx=5, pady=(5, 0)) + ttk.Button( + btn_row1, text="Add Selected", command=self._player_add_selection_to_temp + ).pack(side="left", expand=True, fill="x") + ttk.Button( + btn_row1, + text="Remove Selected", + command=self._player_remove_selected_from_temp, + ).pack(side="left", expand=True, fill="x", padx=(5, 0)) + + btn_row2 = ttk.Frame(playlist_box) + btn_row2.pack(fill="x", padx=5, pady=(5, 5)) + ttk.Button(btn_row2, text="Clear", command=self._player_clear_temp_playlist).pack( + side="left", fill="x" + ) + ttk.Button( + btn_row2, text="Save Playlist", command=self._player_save_temp_playlist + ).pack(side="left", expand=True, fill="x", padx=(5, 0)) + ttk.Button( + btn_row2, + text="Current Playlists", + command=self._player_show_current_playlists, + ).pack(side="left", expand=True, fill="x", padx=(5, 0)) + self._sync_player_playlist() + # ─── Library Sync Tab ───────────────────────────────────────────── self.sync_tab = ttk.Frame(self.notebook) self.notebook.add(self.sync_tab, text="Library Sync") @@ -1883,6 +1935,8 @@ def select_library(self): self._load_genre_mapping() if hasattr(self, "player_search_var"): self.player_search_var.set("") + if hasattr(self, "player_playlist_listbox"): + self._player_set_temp_playlist([]) # Clear any cached clustering data when switching libraries self.cluster_data = None self.cluster_manager = None @@ -3356,6 +3410,142 @@ def _update_player_art( caption = "Select a track to view album art" self.player_art_caption.configure(text=caption, wraplength=size + 20) + def _get_selected_player_paths(self) -> list[str]: + if not hasattr(self, "player_tree"): + return [] + selection = self.player_tree.selection() + return [ + self.player_tree_paths[item] + for item in selection + if item in self.player_tree_paths + ] + + def _player_set_temp_playlist(self, tracks: list[str]) -> None: + self.player_temp_playlist = list(dict.fromkeys(tracks)) + self._sync_player_playlist() + + def _sync_player_playlist(self) -> None: + if self.player_playlist_listbox is None: + return + self.player_playlist_listbox.delete(0, tk.END) + for path in self.player_temp_playlist: + self.player_playlist_listbox.insert(tk.END, os.path.basename(path)) + + count = len(self.player_temp_playlist) + suffix = "song" if count == 1 else "songs" + self.player_playlist_status_var.set(f"Playlist builder items: {count} {suffix}") + + def _player_add_selection_to_temp(self) -> None: + selected_paths = self._get_selected_player_paths() + if not selected_paths: + messagebox.showinfo( + "Playlist Builder", "Select one or more songs to add first." + ) + return + + combined = list(self.player_temp_playlist) + for path in selected_paths: + if path not in combined: + combined.append(path) + self._player_set_temp_playlist(combined) + + def _player_remove_selected_from_temp(self) -> None: + if self.player_playlist_listbox is None: + return + selected = list(self.player_playlist_listbox.curselection()) + if not selected: + messagebox.showinfo("Playlist Builder", "Select songs in the list to remove.") + return + + remaining = [ + track for i, track in enumerate(self.player_temp_playlist) if i not in selected + ] + self._player_set_temp_playlist(remaining) + + def _player_clear_temp_playlist(self) -> None: + if not self.player_temp_playlist: + return + self._player_set_temp_playlist([]) + + def _player_save_temp_playlist(self) -> None: + if not self.player_temp_playlist: + messagebox.showinfo("Playlist Builder", "Add songs before saving a playlist.") + return + if not self.library_path: + messagebox.showwarning("No Library", "Select a library before saving.") + return + + outfile = self._player_prompt_playlist_destination() + if not outfile: + return + + try: + write_playlist(self.player_temp_playlist, outfile) + except Exception as exc: + messagebox.showerror("Playlist", f"Failed to save playlist: {exc}") + return + + messagebox.showinfo("Playlist", f"Playlist saved to {outfile}") + self._open_folder(os.path.dirname(outfile)) + + def _player_prompt_playlist_destination(self) -> str | None: + playlists_dir = os.path.join(self.library_path, "Playlists") + os.makedirs(playlists_dir, exist_ok=True) + ts = datetime.now().strftime("%Y%m%d_%H%M%S") + default_name = f"PlayerSelection_{ts}.m3u" + + chosen = filedialog.asksaveasfilename( + parent=self, + title="Save Playlist As", + defaultextension=".m3u", + initialdir=playlists_dir, + initialfile=default_name, + filetypes=[("M3U Playlist", "*.m3u"), ("All Files", "*.*")], + ) + + if not chosen: + return None + + return os.path.join(playlists_dir, os.path.basename(chosen)) + + def _player_show_current_playlists(self) -> None: + if not self.library_path: + messagebox.showwarning("No Library", "Select a library first.") + return + + playlists_dir = os.path.join(self.library_path, "Playlists") + if not os.path.isdir(playlists_dir): + messagebox.showinfo( + "Playlists", "No playlists folder found yet. Save one to get started." + ) + return + try: + entries = [ + f + for f in os.listdir(playlists_dir) + if os.path.isfile(os.path.join(playlists_dir, f)) + ] + except OSError as exc: + messagebox.showerror("Playlists", f"Could not read folder: {exc}") + return + + if not entries: + messagebox.showinfo("Playlists", "No playlists found.") + return + + messagebox.showinfo("Playlists", "\n".join(sorted(entries))) + + def _open_folder(self, path: str) -> None: + try: + if sys.platform == "win32": + os.startfile(path) # type: ignore[attr-defined] + elif sys.platform == "darwin": + subprocess.run(["open", path], check=False) + else: + subprocess.run(["xdg-open", path], check=False) + except Exception as exc: + messagebox.showerror("Open Folder", f"Could not open {path}: {exc}") + def populate_quality_table(self, matches): self.clear_quality_view() self._quality_play_buttons = [] From 99a59c20da2dae8e9f8d1780597267b8c3324810 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 15 Dec 2025 18:14:24 -0500 Subject: [PATCH 046/606] Add scrolling to tempo/energy bucket view --- main_gui.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/main_gui.py b/main_gui.py index b767ac9..87d6158 100644 --- a/main_gui.py +++ b/main_gui.py @@ -721,7 +721,21 @@ def poll_queue(): if running.get(): frame.after(200, poll_queue) - info = ttk.LabelFrame(frame, text="Tempo/Energy Buckets") + canvas = tk.Canvas(frame) + scrollbar = ttk.Scrollbar(frame, orient="vertical", command=canvas.yview) + content = ttk.Frame(canvas) + + content.bind( + "", + lambda e: canvas.configure(scrollregion=canvas.bbox("all")), + ) + canvas.create_window((0, 0), window=content, anchor="nw") + canvas.configure(yscrollcommand=scrollbar.set) + + canvas.pack(side="left", fill="both", expand=True) + scrollbar.pack(side="right", fill="y") + + info = ttk.LabelFrame(content, text="Tempo/Energy Buckets") info.pack(fill="x", padx=10, pady=(5, 10)) ttk.Label(info, textvariable=lib_var).pack(anchor="w", padx=5, pady=(5, 0)) ttk.Label(info, textvariable=dep_status, wraplength=360).pack( @@ -752,7 +766,7 @@ def poll_queue(): link.pack(anchor="w", padx=5, pady=(0, 5)) link.bind("", open_instructions) - controls = ttk.Frame(frame) + controls = ttk.Frame(content) controls.pack(fill="x", padx=10) run_btn = ttk.Button(controls, text="Generate Buckets", command=start_run) run_btn.pack(side="left") @@ -766,16 +780,16 @@ def poll_queue(): command=update_controls, ).pack(side="right") - progress = ttk.Progressbar(frame, mode="determinate") + progress = ttk.Progressbar(content, mode="determinate") progress.pack(fill="x", padx=10, pady=(10, 0)) - ttk.Label(frame, textvariable=progress_var).pack(anchor="w", padx=12) + ttk.Label(content, textvariable=progress_var).pack(anchor="w", padx=12) - log_group = ttk.LabelFrame(frame, text="Run Log") + log_group = ttk.LabelFrame(content, text="Run Log") log_group.pack(fill="both", expand=False, padx=10, pady=(10, 0)) log_box = ScrolledText(log_group, height=8, state="disabled") log_box.pack(fill="both", expand=True, padx=5, pady=5) - stats_group = ttk.LabelFrame(frame, text="Bucket Summary") + stats_group = ttk.LabelFrame(content, text="Bucket Summary") stats_group.pack(fill="both", expand=True, padx=10, pady=(10, 10)) stats_rows = ttk.Frame(stats_group) stats_rows.pack(fill="both", expand=True) From 05ed8375458701bbb90bc9b0c3cb9f9ce80e0bff Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 15 Dec 2025 18:24:19 -0500 Subject: [PATCH 047/606] Fix tempo bucket panel resizing with scrollbar --- main_gui.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/main_gui.py b/main_gui.py index 87d6158..a0aedec 100644 --- a/main_gui.py +++ b/main_gui.py @@ -729,7 +729,11 @@ def poll_queue(): "", lambda e: canvas.configure(scrollregion=canvas.bbox("all")), ) - canvas.create_window((0, 0), window=content, anchor="nw") + content_window = canvas.create_window((0, 0), window=content, anchor="nw") + canvas.bind( + "", + lambda e: canvas.itemconfigure(content_window, width=e.width), + ) canvas.configure(yscrollcommand=scrollbar.set) canvas.pack(side="left", fill="both", expand=True) From 10913aa8c62cfbaa8b04b0f28a6c8ce457f2823c Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 15 Dec 2025 18:25:46 -0500 Subject: [PATCH 048/606] Handle clustering progress in interactive clustering tools --- main_gui.py | 62 +++++++++++++++++++++++++++++++++-------------------- 1 file changed, 39 insertions(+), 23 deletions(-) diff --git a/main_gui.py b/main_gui.py index 87d6158..d6c3f7b 100644 --- a/main_gui.py +++ b/main_gui.py @@ -866,12 +866,16 @@ def generate(): if tracks is None: msg = ttk.Frame(frame) msg.pack(padx=10, pady=10) - ttk.Label(msg, text="Run clustering once first").pack() - ttk.Button( - msg, - text="Run Clustering", - command=lambda: app.cluster_playlists_dialog(params["method"]), - ).pack(pady=(5, 0)) + if getattr(app, "cluster_generation_running", False): + ttk.Label(msg, text="Clustering in progress…").pack() + ttk.Label(msg, text="Check the log tab for updates.").pack() + else: + ttk.Label(msg, text="Run clustering once first").pack() + ttk.Button( + msg, + text="Run Clustering", + command=lambda: app.cluster_playlists_dialog(params["method"]), + ).pack(pady=(5, 0)) return frame # Container ensures the graph resizes while keeping controls visible @@ -2521,6 +2525,8 @@ def _start_cluster_playlists(self, method: str, params: dict, engine: str, dlg): path = self.require_library() if not path: return + self.cluster_generation_running = True + self._refresh_plugin_panel() self.show_log_tab() threading.Thread( target=self._run_cluster_generation, @@ -2531,26 +2537,36 @@ def _start_cluster_playlists(self, method: str, params: dict, engine: str, dlg): def _run_cluster_generation( self, path: str, method: str, params: dict, engine: str ): - tracks, feats = cluster_library( - path, method, params, self._log, self.folder_filter, engine - ) - self.cluster_data = (tracks, feats) - self.cluster_params = {"method": method, "engine": engine, **params} - self.cluster_manager = ClusterComputationManager(tracks, feats, self._log) + try: + tracks, feats = cluster_library( + path, method, params, self._log, self.folder_filter, engine + ) + self.cluster_data = (tracks, feats) + self.cluster_params = {"method": method, "engine": engine, **params} + self.cluster_manager = ClusterComputationManager(tracks, feats, self._log) - for name in ("Interactive – KMeans", "Interactive – HDBSCAN"): - if name in self.plugin_views: - try: - self.plugin_views[name].destroy() - except Exception: - pass - self.plugin_views.pop(name, None) + for name in ("Interactive – KMeans", "Interactive – HDBSCAN"): + if name in self.plugin_views: + try: + self.plugin_views[name].destroy() + except Exception: + pass + self.plugin_views.pop(name, None) + + def done(): + self.cluster_generation_running = False + messagebox.showinfo("Clustered Playlists", "Generation complete") + self._refresh_plugin_panel() - def done(): - messagebox.showinfo("Clustered Playlists", "Generation complete") - self._refresh_plugin_panel() + self.after(0, done) + except Exception as exc: + def fail(): + self.cluster_generation_running = False + messagebox.showerror("Cluster Generation Failed", str(exc)) + self._log(f"✘ Cluster generation failed: {exc}") + self._refresh_plugin_panel() - self.after(0, done) + self.after(0, fail) def _refresh_plugin_panel(self): """Rebuild the current plugin panel if a plugin is selected.""" From b4bd1e2a83f8c66b63572bd2c7cd36bd49a0345f Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 15 Dec 2025 18:29:55 -0500 Subject: [PATCH 049/606] Cache tempo energy bucket analysis --- playlist_engine.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/playlist_engine.py b/playlist_engine.py index 7277f7e..3001d40 100644 --- a/playlist_engine.py +++ b/playlist_engine.py @@ -116,14 +116,35 @@ def bucket_by_tempo_energy( ) playlists_dir = os.path.join(root_path, "Playlists") os.makedirs(playlists_dir, exist_ok=True) + + # Cached results setup (mirrors clustering tools for faster re-runs) + docs_dir = os.path.join(root_path, "Docs") + os.makedirs(docs_dir, exist_ok=True) + cache_file = os.path.join(docs_dir, "tempo_energy.npy") + try: + cache = dict(np.load(cache_file, allow_pickle=True).item()) + log_callback(f"→ Loaded {len(cache)} cached tempo/energy entries") + except FileNotFoundError: + cache = {} + log_callback("→ No tempo/energy cache found; analyzing all tracks") + buckets: dict[tuple[str, str], list[str]] = {} processed = 0 + updated_cache = False for path in tracks: if cancel_event and cancel_event.is_set(): log_callback("! Bucket generation cancelled by user.") break + cached = cache.get(path) try: - tempo, rms = compute_tempo_energy(path, engine=engine) + if cached and cached.get("engine") == engine: + tempo = float(cached["tempo"]) + rms = float(cached["rms"]) + log_callback(f"• Using cached tempo/energy for {os.path.basename(path)}") + else: + tempo, rms = compute_tempo_energy(path, engine=engine) + cache[path] = {"engine": engine, "tempo": tempo, "rms": rms} + updated_cache = True tb = categorize_tempo(tempo) eb = categorize_energy(rms) buckets.setdefault((tb, eb), []).append(path) @@ -132,6 +153,10 @@ def bucket_by_tempo_energy( log_callback(f"! Failed analysis for {path}: {e}") processed += 1 progress_callback(processed) + + if updated_cache: + np.save(cache_file, cache) + log_callback(f"✓ Saved tempo/energy cache ({len(cache)} entries) to {cache_file}") out_paths = {} for (tb, eb), items in buckets.items(): outfile = os.path.join(playlists_dir, f"{tb}_{eb}.m3u") From b7e53bd73d98785e9687b2afbaacdb8d62d1b1e6 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 15 Dec 2025 18:38:45 -0500 Subject: [PATCH 050/606] Remove unused unique genre lister tool --- README.md | 1 - controllers/genre_list_controller.py | 85 ------------------------- docs/project_documentation.html | 92 +--------------------------- main_gui.py | 5 -- 4 files changed, 2 insertions(+), 181 deletions(-) delete mode 100644 controllers/genre_list_controller.py diff --git a/README.md b/README.md index d4f7c04..7b0358e 100644 --- a/README.md +++ b/README.md @@ -166,7 +166,6 @@ controllers/ cluster_controller.py - Gather tracks and run clustering library_index_controller.py - Build HTML index of your library highlight_controller.py - Play short audio snippets - genre_list_controller.py - Scan library for unique genres playlist_controller.py - Playlist export placeholder plugins/ diff --git a/controllers/genre_list_controller.py b/controllers/genre_list_controller.py deleted file mode 100644 index 71082e0..0000000 --- a/controllers/genre_list_controller.py +++ /dev/null @@ -1,85 +0,0 @@ -import os -from tkinter import messagebox -from typing import Set -from mutagen import File as MutagenFile - -SUPPORTED_EXTS = {".flac", ".m4a", ".aac", ".mp3", ".wav", ".ogg"} - - -def sanitize(text: str) -> str: - if not text: - return "" - return ( - text.replace("&", "&") - .replace("<", "<") - .replace(">", ">") - ) - - -def extract_genre(filepath: str) -> str | None: - try: - audio = MutagenFile(filepath, easy=True) - if not audio or not audio.tags: - return None - raw_genre = audio.tags.get("genre", [None])[0] - if raw_genre: - return raw_genre.strip() - return None - except Exception: - return None - - -def gather_unique_genres(root_path: str) -> Set[str]: - unique: Set[str] = set() - for dirpath, _, filenames in os.walk(root_path): - for fname in filenames: - ext = os.path.splitext(fname)[1].lower() - if ext in SUPPORTED_EXTS: - fullpath = os.path.join(dirpath, fname) - g = extract_genre(fullpath) - if g: - unique.add(g) - return unique - - -def write_genres_html(root_path: str, genres: Set[str], output_filename: str = "genres.html") -> str: - sorted_genres = sorted(genres, key=lambda s: s.lower()) - lines = [] - lines.append("") - lines.append("") - lines.append("") - lines.append(" ") - lines.append(f" Unique Genres in {sanitize(os.path.basename(root_path) or root_path)}") - lines.append(" ") - lines.append("") - lines.append("") - lines.append(f"

      Unique Genres in {sanitize(os.path.basename(root_path) or root_path)}

      ") - lines.append("
        ") - if sorted_genres: - for genre in sorted_genres: - lines.append(f"
      • {sanitize(genre)}
      • ") - else: - lines.append("
      • No genre tags found
      • ") - lines.append("
      ") - lines.append("") - lines.append("") - output_path = os.path.join(root_path, output_filename) - with open(output_path, "w", encoding="utf-8") as f: - f.write("\n".join(lines)) - return output_path - - -def list_unique_genres(folder_path: str) -> str: - """Scan ``folder_path`` and write genres.html listing unique genres.""" - genres = gather_unique_genres(folder_path) - output_path = write_genres_html(folder_path, genres) - messagebox.showinfo( - "Genre List Generated", - f"Your genre list has been saved to:\n{output_path}" - ) - return output_path diff --git a/docs/project_documentation.html b/docs/project_documentation.html index 66cebfb..ba7bec0 100644 --- a/docs/project_documentation.html +++ b/docs/project_documentation.html @@ -150,94 +150,7 @@

      2.3 Fingerprint Cache (fingerprint_cache.py)


      -

      2.4 Unique Genre Lister (genre_list_controller.py)

      -

      Scans a folder for audio files and builds a deduplicated, sorted list of all genre tags found. Writes genres.html so you can review every raw genre:

      - -
      
      -#!/usr/bin/env python3
      -"""
      -genre_list_controller.py
      -
      -Scan audio files, extract their genre tags, deduplicate them and output genres.html.
      -"""
      -
      -import os
      -import sys
      -import tkinter as tk
      -from tkinter import filedialog
      -from mutagen import File as MutagenFile
      -
      -SUPPORTED_EXTS = {".flac", ".m4a", ".aac", ".mp3", ".wav", ".ogg"}
      -
      -def sanitize(text: str) -> str:
      -    # Escape HTML: <, >, &
      -    return text.replace("&", "&").replace("<", "<").replace(">", ">")
      -
      -def extract_genre(filepath: str) -> str:
      -    """
      -    Return the first 'genre' tag (string) from the audio file, or None.
      -    """
      -    try:
      -        audio = MutagenFile(filepath, easy=True)
      -        if not audio or not audio.tags:
      -            return None
      -        raw = audio.tags.get("genre", [None])[0]
      -        if raw:
      -            return raw.strip()
      -        return None
      -    except Exception:
      -        return None
      -
      -def gather_unique_genres(root_path: str) -> set:
      -    unique = set()
      -    for dirpath, _, filenames in os.walk(root_path):
      -        for fname in filenames:
      -            ext = os.path.splitext(fname)[1].lower()
      -            if ext in SUPPORTED_EXTS:
      -                fullpath = os.path.join(dirpath, fname)
      -                genre = extract_genre(fullpath)
      -                if genre:
      -                    unique.add(genre)
      -    return unique
      -
      -def write_genres_html(root_path: str, genres: set, output_filename: str = "genres.html"):
      -    sorted_genres = sorted(genres, key=lambda s: s.lower())
      -    lines = []
      -    lines.append("")
      -    lines.append("")
      -    lines.append("Unique Genres")
      -    lines.append("")
      -    lines.append("")
      -    lines.append(f"

      Unique Genres in “{sanitize(os.path.basename(root_path) or root_path)}”

      ") - lines.append("
        ") - if sorted_genres: - for genre in sorted_genres: - lines.append(f"
      • {sanitize(genre)}
      • ") - else: - lines.append("
      • No genre tags found
      • ") - lines.append("
      ") - lines.append("") - - out_path = os.path.join(root_path, output_filename) - with open(out_path, "w", encoding="utf-8") as f: - f.write("\n".join(lines)) - -def main(): - root = tk.Tk() - root.withdraw() - folder = filedialog.askdirectory(title="Select Folder to Scan for Genres") - if not folder: - sys.exit(0) - genres = gather_unique_genres(folder) - write_genres_html(folder, genres) - -if __name__ == "__main__": - main() -
      - -
      - -

      2.5 Genre Updater via MusicBrainz (update_genres.py)

      +

      2.4 Genre Updater via MusicBrainz (update_genres.py)

      Walks your music folder, reads each file’s artist and title, queries MusicBrainz for a matching recording, grabs the top 3 tags by popularity, and writes them back into each file’s genre field. Logs all actions into genre_update_log.txt. Rate-limited at 1 request/second.

      
      @@ -624,8 +537,8 @@ 

      3. Current Progress

    • Indexer: Fully implemented as music_indexer_api.py, tested on sample folders. It correctly reorganizes by artist, year, album, singles, and remixes, preserving cover-art groups.
    • Fingerprint Generator: fingerprint_generator.py stores AcoustID hashes for each track.
    • Fingerprint Cache: fingerprint_cache.py keeps computed fingerprints in SQLite.
    • -
    • Unique Genre Lister: genre_list_controller.py creates genres.html showing all distinct raw genre tags in one place.
    • Genre Updater: update_genres.py connects to MusicBrainz, pulls top 3 tags per track, and writes them into files—skipping files that already have ≥ 2 genres.
    • +
    • Genre Normalizer: normalize_controller.py scans raw genre tags, suggests clean mappings, and writes normalized genres back to your files.
    • Library Index: library_index_controller.py builds library_index.html, a table listing path, artist, title, album, year, track, and genre(s) for every audio file. Useful for quick review.
    • Near-Duplicate Detector: integrated into music_indexer_api.py to remove fuzzy duplicates.
    • Clustered Playlists: interactive K-Means and HDBSCAN clustering with a graph-based panel.
    • @@ -719,7 +632,6 @@

      5. Summary & Migration Tips

      gui/ main_gui.py controllers/ - genre_list_controller.py library_index_controller.py import_controller.py tagfix_controller.py diff --git a/main_gui.py b/main_gui.py index f2a5f30..657a215 100644 --- a/main_gui.py +++ b/main_gui.py @@ -44,7 +44,6 @@ import fingerprint_cache import chromaprint_utils from controllers.library_index_controller import generate_index -from controllers.genre_list_controller import list_unique_genres from controllers.scan_progress_controller import ScanProgressController from gui.audio_preview import PlaybackError, VlcPreviewPlayer from io import BytesIO @@ -1338,10 +1337,6 @@ def build_ui(self): label="Generate Library Index…", command=lambda: generate_index(self.require_library()), ) - tools_menu.add_command( - label="List Unique Genres…", - command=lambda: list_unique_genres(self.require_library()), - ) tools_menu.add_separator() tools_menu.add_command(label="Reset Tag-Fix Log", command=self.reset_tagfix_log) menubar.add_cascade(label="Tools", menu=tools_menu) From 0be2d7ea38aff8247a4df0c190013a4c7ba92763 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 15 Dec 2025 18:40:27 -0500 Subject: [PATCH 051/606] Handle non-string paths in tempo bucket analysis --- playlist_engine.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/playlist_engine.py b/playlist_engine.py index 3001d40..89f9c2e 100644 --- a/playlist_engine.py +++ b/playlist_engine.py @@ -132,6 +132,17 @@ def bucket_by_tempo_energy( processed = 0 updated_cache = False for path in tracks: + try: + path = os.fspath(path) + except TypeError: + # Handle unexpected iterable inputs (e.g., lists from GUI selections) + if isinstance(path, (list, tuple)) and path: + path = os.fspath(path[0]) + else: + log_callback(f"! Skipping unsupported path value: {path!r}") + processed += 1 + progress_callback(processed) + continue if cancel_event and cancel_event.is_set(): log_callback("! Bucket generation cancelled by user.") break From 59315fe98b81da162fe7d12bb580a33939a879b7 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 15 Dec 2025 18:46:50 -0500 Subject: [PATCH 052/606] Restrict clustering prompt to clustering tools --- main_gui.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/main_gui.py b/main_gui.py index 657a215..f6ad954 100644 --- a/main_gui.py +++ b/main_gui.py @@ -145,6 +145,8 @@ def create_panel_for_plugin(app, name: str, parent: tk.Widget) -> ttk.Frame: app.cluster_manager = ClusterComputationManager(tracks, features, app._log) cluster_manager = app.cluster_manager + requires_clustering = name in {"Interactive – KMeans", "Interactive – HDBSCAN"} + if name == "Interactive – KMeans": from sklearn.cluster import KMeans @@ -866,7 +868,7 @@ def generate(): ttk.Label(frame, text=f"{name} panel coming soon…").pack(padx=10, pady=10) return frame - if tracks is None: + if requires_clustering and tracks is None: msg = ttk.Frame(frame) msg.pack(padx=10, pady=10) if getattr(app, "cluster_generation_running", False): From 777db2405ae40d33840f57bc6c8345e1c4cc6ba7 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 15 Dec 2025 18:49:46 -0500 Subject: [PATCH 053/606] Normalize tempo cache paths and skip re-analysis when available --- playlist_engine.py | 44 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/playlist_engine.py b/playlist_engine.py index 89f9c2e..51ecdfe 100644 --- a/playlist_engine.py +++ b/playlist_engine.py @@ -128,40 +128,64 @@ def bucket_by_tempo_energy( cache = {} log_callback("→ No tempo/energy cache found; analyzing all tracks") + # Normalize cache keys to absolute paths so repeated runs can reuse results + # even if callers pass in different path representations. + normalized_cache: dict[str, dict] = {} + for key, value in cache.items(): + abs_key = os.path.abspath(os.fspath(key)) + normalized_cache[abs_key] = value + cache_normalized_changed = cache != normalized_cache + cache = normalized_cache + buckets: dict[tuple[str, str], list[str]] = {} processed = 0 - updated_cache = False + updated_cache = cache_normalized_changed + + normalized_tracks: list[tuple[str, str]] = [] for path in tracks: try: - path = os.fspath(path) + path_str = os.fspath(path) except TypeError: # Handle unexpected iterable inputs (e.g., lists from GUI selections) if isinstance(path, (list, tuple)) and path: - path = os.fspath(path[0]) + path_str = os.fspath(path[0]) else: log_callback(f"! Skipping unsupported path value: {path!r}") processed += 1 progress_callback(processed) continue + normalized_tracks.append((path_str, os.path.abspath(path_str))) + + all_cached = bool(normalized_tracks) and all( + (entry := cache.get(abs_path)) and entry.get("engine") == engine + for _, abs_path in normalized_tracks + ) + if all_cached: + log_callback( + f"→ All {len(normalized_tracks)} tracks already cached with engine='{engine}'. Skipping re-analysis." + ) + + for path_str, cache_key in normalized_tracks: if cancel_event and cancel_event.is_set(): log_callback("! Bucket generation cancelled by user.") break - cached = cache.get(path) + cached = cache.get(cache_key) try: if cached and cached.get("engine") == engine: tempo = float(cached["tempo"]) rms = float(cached["rms"]) - log_callback(f"• Using cached tempo/energy for {os.path.basename(path)}") + if not all_cached: + log_callback(f"• Using cached tempo/energy for {os.path.basename(path_str)}") else: - tempo, rms = compute_tempo_energy(path, engine=engine) - cache[path] = {"engine": engine, "tempo": tempo, "rms": rms} + tempo, rms = compute_tempo_energy(path_str, engine=engine) + cache[cache_key] = {"engine": engine, "tempo": tempo, "rms": rms} updated_cache = True tb = categorize_tempo(tempo) eb = categorize_energy(rms) - buckets.setdefault((tb, eb), []).append(path) - log_callback(f"• {os.path.basename(path)} → {tb}/{eb}") + buckets.setdefault((tb, eb), []).append(path_str) + log_callback(f"• {os.path.basename(path_str)} → {tb}/{eb}") except Exception as e: - log_callback(f"! Failed analysis for {path}: {e}") + log_callback(f"! Failed analysis for {path_str}: {e}") processed += 1 progress_callback(processed) From f59b9fadfaadc10aa80816772435259d884c007e Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 15 Dec 2025 18:59:45 -0500 Subject: [PATCH 054/606] Keep clustering controls visible for interactive graphs --- main_gui.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/main_gui.py b/main_gui.py index f6ad954..ece163c 100644 --- a/main_gui.py +++ b/main_gui.py @@ -868,20 +868,28 @@ def generate(): ttk.Label(frame, text=f"{name} panel coming soon…").pack(padx=10, pady=10) return frame - if requires_clustering and tracks is None: + if requires_clustering: msg = ttk.Frame(frame) - msg.pack(padx=10, pady=10) + msg.pack(fill="x", padx=10, pady=10) + + run_btn = ttk.Button( + msg, + text="Run Clustering", + command=lambda: app.cluster_playlists_dialog(params["method"]), + ) + run_btn.pack(side="left") + + status = ttk.Label(msg) + status.pack(side="left", padx=(8, 0)) + if getattr(app, "cluster_generation_running", False): - ttk.Label(msg, text="Clustering in progress…").pack() - ttk.Label(msg, text="Check the log tab for updates.").pack() + status.config(text="Clustering in progress…") + run_btn.state(["disabled"]) + elif tracks is None: + status.config(text="Run clustering once first") + return frame else: - ttk.Label(msg, text="Run clustering once first").pack() - ttk.Button( - msg, - text="Run Clustering", - command=lambda: app.cluster_playlists_dialog(params["method"]), - ).pack(pady=(5, 0)) - return frame + status.config(text="Clusters ready") # Container ensures the graph resizes while keeping controls visible container = ttk.Frame(frame) From 49f73d866c94a6bbd0326950c5e99ee0a991a843 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 15 Dec 2025 19:06:44 -0500 Subject: [PATCH 055/606] Reserve clustering toolbar space without showing run button --- main_gui.py | 40 +++++++++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/main_gui.py b/main_gui.py index f6ad954..2486527 100644 --- a/main_gui.py +++ b/main_gui.py @@ -868,20 +868,38 @@ def generate(): ttk.Label(frame, text=f"{name} panel coming soon…").pack(padx=10, pady=10) return frame - if requires_clustering and tracks is None: + if requires_clustering: msg = ttk.Frame(frame) - msg.pack(padx=10, pady=10) + msg.pack(fill="x", padx=10, pady=10) + + def _reserve_button_width(btn: ttk.Button) -> None: + btn.pack(side="left") + msg.update_idletasks() + width = btn.winfo_reqwidth() + btn.pack_forget() + ttk.Frame(msg, width=width).pack(side="left") + + run_btn = ttk.Button( + msg, + text="Run Clustering", + command=lambda: app.cluster_playlists_dialog(params["method"]), + ) + + status = ttk.Label(msg) + if getattr(app, "cluster_generation_running", False): - ttk.Label(msg, text="Clustering in progress…").pack() - ttk.Label(msg, text="Check the log tab for updates.").pack() + _reserve_button_width(run_btn) + status.config(text="Clustering in progress…") + status.pack(side="left", padx=(8, 0)) + elif tracks is None: + run_btn.pack(side="left") + status.config(text="Run clustering once first") + status.pack(side="left", padx=(8, 0)) + return frame else: - ttk.Label(msg, text="Run clustering once first").pack() - ttk.Button( - msg, - text="Run Clustering", - command=lambda: app.cluster_playlists_dialog(params["method"]), - ).pack(pady=(5, 0)) - return frame + _reserve_button_width(run_btn) + status.config(text="Clusters ready") + status.pack(side="left", padx=(8, 0)) # Container ensures the graph resizes while keeping controls visible container = ttk.Frame(frame) From 296333117225e5d3c66733e163279d2550c7c89e Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 15 Dec 2025 19:15:56 -0500 Subject: [PATCH 056/606] Add save options for tempo/energy bucket playlists --- main_gui.py | 52 ++++++++++++++++++++++++++++++++++++++++++---- playlist_engine.py | 6 +++++- 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/main_gui.py b/main_gui.py index ece163c..090f058 100644 --- a/main_gui.py +++ b/main_gui.py @@ -544,6 +544,8 @@ def open_genre_folder(): running = tk.BooleanVar(value=False) total_tracks = 0 playlists_dir: str | None = None + bucket_result: dict | None = None + bucket_tracks: dict[tuple[str, str], list[str]] = {} q: queue.Queue = queue.Queue() cancel_event = threading.Event() engine_var = getattr(app, "feature_engine_var", tk.StringVar(value="librosa")) @@ -602,7 +604,43 @@ def append_log(msg: str) -> None: log_box.see("end") log_box.configure(state="disabled") - def render_stats(stats: dict | None): + def prompt_save_bucket(bucket_key: tuple[str, str]): + tracks = bucket_tracks.get(bucket_key) + if not tracks: + messagebox.showinfo( + "Save Playlist", + "No tracks available for this bucket yet.", + ) + return + + playlists_root = playlists_dir or app.library_path or os.getcwd() + os.makedirs(playlists_root, exist_ok=True) + default_name = f"{bucket_key[0]}_{bucket_key[1]}.m3u" + chosen = filedialog.asksaveasfilename( + parent=frame, + title="Save Bucket Playlist As", + defaultextension=".m3u", + initialdir=playlists_root, + initialfile=default_name, + filetypes=[("M3U Playlist", "*.m3u"), ("All Files", "*.*")], + ) + if not chosen: + return + + try: + write_playlist(tracks, chosen) + except Exception as exc: + messagebox.showerror("Save Playlist", f"Could not save playlist: {exc}") + return + + messagebox.showinfo("Playlist Saved", f"Playlist saved to {chosen}") + open_path(os.path.dirname(chosen)) + + def render_stats(result: dict | None): + nonlocal bucket_result, bucket_tracks + bucket_result = result + bucket_tracks = (result or {}).get("buckets", {}) if isinstance(result, dict) else {} + stats = (result or {}).get("stats") if isinstance(result, dict) else None for child in stats_rows.winfo_children(): child.destroy() if not stats: @@ -622,13 +660,19 @@ def render_stats(stats: dict | None): ttk.Label(row, text=tb.title(), width=10).pack(side="left") ttk.Label(row, text=eb.title(), width=10).pack(side="left") ttk.Label(row, text=str(info.get("count", 0)), width=8).pack(side="left") - ttk.Label(row, text=os.path.basename(info.get("playlist", ""))).pack( + playlist_path = info.get("playlist", "") + ttk.Label(row, text=os.path.basename(playlist_path)).pack( side="left", padx=(10, 5) ) ttk.Button( row, text="Open", - command=lambda p=info.get("playlist", ""): open_path(p), + command=lambda p=playlist_path: open_path(p), + ).pack(side="left", padx=(0, 5)) + ttk.Button( + row, + text="Save As…", + command=lambda key=(tb, eb): prompt_save_bucket(key), ).pack(side="left") def update_controls(): @@ -696,7 +740,7 @@ def poll_queue(): progress_var.set(f"{payload}/{total_tracks} processed") elif tag == "done": running.set(False) - render_stats(payload.get("buckets")) + render_stats(payload) status = ( "Cancelled" if payload.get("cancelled") diff --git a/playlist_engine.py b/playlist_engine.py index 51ecdfe..c525eb1 100644 --- a/playlist_engine.py +++ b/playlist_engine.py @@ -199,7 +199,11 @@ def bucket_by_tempo_energy( out_paths[(tb, eb)] = outfile log_callback(f"→ Wrote {outfile}") stats = { - (tb, eb): {"count": len(items), "playlist": out_paths[(tb, eb)]} + (tb, eb): { + "count": len(items), + "playlist": out_paths[(tb, eb)], + "tracks": list(items), + } for (tb, eb), items in buckets.items() } From 96909ffe2f4baf06e690e4341b17537f3475e607 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 15 Dec 2025 19:20:09 -0500 Subject: [PATCH 057/606] Hide run clustering button after clusters render --- main_gui.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/main_gui.py b/main_gui.py index 090f058..0becc28 100644 --- a/main_gui.py +++ b/main_gui.py @@ -912,7 +912,7 @@ def generate(): ttk.Label(frame, text=f"{name} panel coming soon…").pack(padx=10, pady=10) return frame - if requires_clustering: + if requires_clustering and tracks is None: msg = ttk.Frame(frame) msg.pack(fill="x", padx=10, pady=10) @@ -929,11 +929,9 @@ def generate(): if getattr(app, "cluster_generation_running", False): status.config(text="Clustering in progress…") run_btn.state(["disabled"]) - elif tracks is None: - status.config(text="Run clustering once first") - return frame else: - status.config(text="Clusters ready") + status.config(text="Run clustering once first") + return frame # Container ensures the graph resizes while keeping controls visible container = ttk.Frame(frame) From 12f220befbeaec05b53cb8566926b08693023891 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 15 Dec 2025 19:24:58 -0500 Subject: [PATCH 058/606] Force graph canvas layout after clustering completes --- cluster_graph_panel.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cluster_graph_panel.py b/cluster_graph_panel.py index 77e8a72..31e4e87 100644 --- a/cluster_graph_panel.py +++ b/cluster_graph_panel.py @@ -390,6 +390,16 @@ def _draw_clusters(self, labels): self.scatter.set_color(point_colors) self.canvas.draw_idle() + def _finalize_graph_layout(self): + """Force geometry updates so the canvas sizes itself correctly.""" + + if not self.canvas.get_tk_widget().winfo_exists(): + return + + widget = self.canvas.get_tk_widget() + widget.update_idletasks() + widget.event_generate("") + def _on_clusters_ready(self, labels, params: dict): duration_ms = None if self._cluster_start_ts is not None: @@ -398,6 +408,7 @@ def _on_clusters_ready(self, labels, params: dict): self._clusters_ready = True if self._loading_lbl.winfo_exists(): self._loading_lbl.pack_forget() + self.after(0, self._finalize_graph_layout) self._draw_clusters(labels) log_cluster_summary(labels, self.log) if duration_ms is not None: From b47ee46f2aa9400d5af775833291e89d583cc8f1 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 15 Dec 2025 19:27:05 -0500 Subject: [PATCH 059/606] Revert "Force graph canvas layout after clustering completes" --- cluster_graph_panel.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/cluster_graph_panel.py b/cluster_graph_panel.py index 31e4e87..77e8a72 100644 --- a/cluster_graph_panel.py +++ b/cluster_graph_panel.py @@ -390,16 +390,6 @@ def _draw_clusters(self, labels): self.scatter.set_color(point_colors) self.canvas.draw_idle() - def _finalize_graph_layout(self): - """Force geometry updates so the canvas sizes itself correctly.""" - - if not self.canvas.get_tk_widget().winfo_exists(): - return - - widget = self.canvas.get_tk_widget() - widget.update_idletasks() - widget.event_generate("") - def _on_clusters_ready(self, labels, params: dict): duration_ms = None if self._cluster_start_ts is not None: @@ -408,7 +398,6 @@ def _on_clusters_ready(self, labels, params: dict): self._clusters_ready = True if self._loading_lbl.winfo_exists(): self._loading_lbl.pack_forget() - self.after(0, self._finalize_graph_layout) self._draw_clusters(labels) log_cluster_summary(labels, self.log) if duration_ms is not None: From da566e6cf82c0446f27d3912128a2119ce0159dc Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 15 Dec 2025 19:44:05 -0500 Subject: [PATCH 060/606] Preview tempo/energy playlists in player --- main_gui.py | 81 ++++++++++++++++++++++++++++++++-------------- playlist_engine.py | 5 +-- 2 files changed, 59 insertions(+), 27 deletions(-) diff --git a/main_gui.py b/main_gui.py index 0becc28..3151055 100644 --- a/main_gui.py +++ b/main_gui.py @@ -636,6 +636,18 @@ def prompt_save_bucket(bucket_key: tuple[str, str]): messagebox.showinfo("Playlist Saved", f"Playlist saved to {chosen}") open_path(os.path.dirname(chosen)) + def preview_bucket(bucket_key: tuple[str, str]) -> None: + tracks = bucket_tracks.get(bucket_key) + if not tracks: + messagebox.showinfo( + "Playlist Preview", "No tracks available for this bucket yet." + ) + return + + app.preview_tracks_in_player( + tracks, f"{bucket_key[0].title()} / {bucket_key[1].title()}" + ) + def render_stats(result: dict | None): nonlocal bucket_result, bucket_tracks bucket_result = result @@ -667,7 +679,7 @@ def render_stats(result: dict | None): ttk.Button( row, text="Open", - command=lambda p=playlist_path: open_path(p), + command=lambda key=(tb, eb): preview_bucket(key), ).pack(side="left", padx=(0, 5)) ttk.Button( row, @@ -749,12 +761,6 @@ def poll_queue(): progress_var.set( f"{status} – processed {payload.get('processed')} of {payload.get('total')} tracks" ) - if ( - not payload.get("cancelled") - and playlists_dir - and payload.get("buckets") - ): - open_path(playlists_dir) update_controls() elif tag == "error": running.set(False) @@ -1339,6 +1345,7 @@ def __init__(self): value="No songs in the playlist builder yet." ) self.player_playlist_listbox: tk.Listbox | None = None + self.player_view_label: str | None = None # assume ffmpeg is available without performing checks self.ffmpeg_available = True @@ -3723,6 +3730,25 @@ def _clear_player_table(self) -> None: self.player_tree_rows.clear() self._update_player_art(None) + def _prepare_player_rows(self, tracks: list[str]) -> list[dict[str, str]]: + rows = [] + for path in tracks: + tags = get_tags(path) + title = tags.get("title") or os.path.basename(path) + artist = tags.get("artist") or "Unknown" + album = tags.get("album") or "" + length_sec = self._get_track_length(path) + rows.append( + { + "title": title, + "artist": artist, + "album": album, + "length": self._format_duration(length_sec), + "path": path, + } + ) + return rows + def _format_duration(self, seconds: float | None) -> str: if not seconds: return "—" @@ -3770,28 +3796,32 @@ def _load_player_library(self, library_path: str) -> None: self.after(0, lambda: self.player_reload_btn.config(state="normal")) return - rows = [] - for path in tracks: - tags = get_tags(path) - title = tags.get("title") or os.path.basename(path) - artist = tags.get("artist") or "Unknown" - album = tags.get("album") or "" - length_sec = self._get_track_length(path) - rows.append( - { - "title": title, - "artist": artist, - "album": album, - "length": self._format_duration(length_sec), - "path": path, - } - ) + rows = self._prepare_player_rows(tracks) self.after(0, lambda: self._update_player_table(rows, library_path)) + def preview_tracks_in_player(self, tracks: list[str], label: str) -> None: + if not tracks: + messagebox.showinfo("Playlist Preview", "No tracks available to preview.") + return + + def worker() -> None: + rows = self._prepare_player_rows(tracks) + self.after(0, lambda: self._show_player_preview(rows, label)) + + threading.Thread(target=worker, daemon=True).start() + + def _show_player_preview(self, rows: list[dict[str, str]], label: str) -> None: + self.player_view_label = f"Previewing {label}" + self.player_tracks = rows + self._apply_player_filter(total_count=len(rows)) + if hasattr(self, "notebook"): + self.notebook.select(self.player_tab) + def _update_player_table(self, rows: list[dict[str, str]], library_path: str) -> None: if library_path != self.library_path: return + self.player_view_label = f"Library – {os.path.basename(library_path)}" self.player_tracks = rows self._apply_player_filter(total_count=len(rows)) @@ -3845,11 +3875,12 @@ def _render_player_rows( total = total_count if total_count is not None else len(rows) shown = len(rows) suffix = "track" if total == 1 else "tracks" + prefix = f"{self.player_view_label}. " if self.player_view_label else "" if not self.preview_backend_available: reason = self.preview_backend_error or "Install python-vlc to enable playback." - status = f"Loaded {total} {suffix}. Preview disabled: {reason}" + status = f"{prefix}Loaded {total} {suffix}. Preview disabled: {reason}" else: - status = f"Loaded {total} {suffix}. Click ▶ for a 30s highlight." + status = f"{prefix}Loaded {total} {suffix}. Click ▶ for a 30s highlight." if shown != total: status += f" Showing {shown} match{'es' if shown != 1 else ''}." self.player_status_var.set(status) diff --git a/playlist_engine.py b/playlist_engine.py index c525eb1..c7ceea1 100644 --- a/playlist_engine.py +++ b/playlist_engine.py @@ -195,9 +195,10 @@ def bucket_by_tempo_energy( out_paths = {} for (tb, eb), items in buckets.items(): outfile = os.path.join(playlists_dir, f"{tb}_{eb}.m3u") - write_playlist(items, outfile) out_paths[(tb, eb)] = outfile - log_callback(f"→ Wrote {outfile}") + log_callback( + f"→ Prepared {len(items)} track(s) for {tb}/{eb} (playlist not yet saved)" + ) stats = { (tb, eb): { "count": len(items), From 0a053ad707e5b23a8030713e8f51edda5b331b1b Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 15 Dec 2025 19:49:53 -0500 Subject: [PATCH 061/606] Ensure cluster graph draws after layout settles --- cluster_graph_panel.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/cluster_graph_panel.py b/cluster_graph_panel.py index 77e8a72..2d1f565 100644 --- a/cluster_graph_panel.py +++ b/cluster_graph_panel.py @@ -398,7 +398,7 @@ def _on_clusters_ready(self, labels, params: dict): self._clusters_ready = True if self._loading_lbl.winfo_exists(): self._loading_lbl.pack_forget() - self._draw_clusters(labels) + self._render_clusters(labels) log_cluster_summary(labels, self.log) if duration_ms is not None: logger.info( @@ -416,6 +416,28 @@ def recluster(self, params: dict): self._loading_lbl.pack(pady=10) self._start_clustering(params) + def _render_clusters(self, labels) -> None: + """Finalize layout before drawing clusters to avoid oversized renders.""" + + def _do_draw() -> None: + layout_owner = self.master if self.master is not None else self + try: + layout_owner.update_idletasks() + except Exception: + pass + try: + self.winfo_toplevel().update_idletasks() + except Exception: + pass + try: + self.canvas.resize_event() + except Exception: + pass + self._draw_clusters(labels) + + # Defer to idle to ensure geometry managers finish placing the widgets + self.after_idle(_do_draw) + def _sync_controls(self): """Enable/disable controls based on clustering readiness.""" state = "normal" if self._clusters_ready else "disabled" From f128859c77c8dfcbfea6d1995f2d928974a892f8 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 15 Dec 2025 19:59:45 -0500 Subject: [PATCH 062/606] Improve playlist graph layout refresh --- main_gui.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/main_gui.py b/main_gui.py index 3151055..b405403 100644 --- a/main_gui.py +++ b/main_gui.py @@ -951,8 +951,13 @@ def generate(): X, X2 = cluster_manager.get_projection() + graph_area = ttk.Frame(container) + graph_area.grid(row=0, column=0, sticky="nsew") + graph_area.rowconfigure(0, weight=1) + graph_area.columnconfigure(0, weight=1) + panel = ClusterGraphPanel( - container, + graph_area, tracks, X, X2, @@ -965,11 +970,18 @@ def generate(): ) panel.grid(row=0, column=0, sticky="nsew", pady=(0, 5)) + # Let Tk settle sizes before showing the side panel so the graph owns space + container.update_idletasks() + side_tools = ttk.Frame(container) side_tools.grid(row=0, column=1, rowspan=3, sticky="ns", padx=(10, 0)) side_tools.columnconfigure(0, weight=1) side_tools.rowconfigure(2, weight=1) + # Once the side panel is present, refresh geometry and redraw the graph + container.update_idletasks() + panel.canvas.draw_idle() + playlist_btn = ttk.Button( side_tools, text="Current Playlists", command=panel.show_current_playlists ) From b173f7e994e90655de6e3fef7faface689072fa4 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 15 Dec 2025 20:21:16 -0500 Subject: [PATCH 063/606] Make clustering side panel persistent --- main_gui.py | 189 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 118 insertions(+), 71 deletions(-) diff --git a/main_gui.py b/main_gui.py index b405403..bf89f74 100644 --- a/main_gui.py +++ b/main_gui.py @@ -918,27 +918,6 @@ def generate(): ttk.Label(frame, text=f"{name} panel coming soon…").pack(padx=10, pady=10) return frame - if requires_clustering and tracks is None: - msg = ttk.Frame(frame) - msg.pack(fill="x", padx=10, pady=10) - - run_btn = ttk.Button( - msg, - text="Run Clustering", - command=lambda: app.cluster_playlists_dialog(params["method"]), - ) - run_btn.pack(side="left") - - status = ttk.Label(msg) - status.pack(side="left", padx=(8, 0)) - - if getattr(app, "cluster_generation_running", False): - status.config(text="Clustering in progress…") - run_btn.state(["disabled"]) - else: - status.config(text="Run clustering once first") - return frame - # Container ensures the graph resizes while keeping controls visible container = ttk.Frame(frame) container.pack(fill="both", expand=True) @@ -946,29 +925,35 @@ def generate(): container.columnconfigure(0, weight=1) container.columnconfigure(1, weight=0) - if cluster_manager is None: - return frame - - X, X2 = cluster_manager.get_projection() - graph_area = ttk.Frame(container) graph_area.grid(row=0, column=0, sticky="nsew") graph_area.rowconfigure(0, weight=1) graph_area.columnconfigure(0, weight=1) - panel = ClusterGraphPanel( - graph_area, - tracks, - X, - X2, - cluster_func=km_func, - cluster_params=params, - cluster_manager=cluster_manager, - algo_key=algo_key, - library_path=app.library_path, - log_callback=app._log, - ) - panel.grid(row=0, column=0, sticky="nsew", pady=(0, 5)) + panel: ClusterGraphPanel | None = None + if cluster_manager is not None: + X, X2 = cluster_manager.get_projection() + + panel = ClusterGraphPanel( + graph_area, + tracks, + X, + X2, + cluster_func=km_func, + cluster_params=params, + cluster_manager=cluster_manager, + algo_key=algo_key, + library_path=app.library_path, + log_callback=app._log, + ) + panel.grid(row=0, column=0, sticky="nsew", pady=(0, 5)) + else: + placeholder = ttk.Label( + graph_area, + text="Run clustering once to view the interactive graph.", + anchor="center", + ) + placeholder.grid(row=0, column=0, sticky="nsew") # Let Tk settle sizes before showing the side panel so the graph owns space container.update_idletasks() @@ -976,44 +961,67 @@ def generate(): side_tools = ttk.Frame(container) side_tools.grid(row=0, column=1, rowspan=3, sticky="ns", padx=(10, 0)) side_tools.columnconfigure(0, weight=1) - side_tools.rowconfigure(2, weight=1) + side_tools.rowconfigure(3, weight=1) # Once the side panel is present, refresh geometry and redraw the graph container.update_idletasks() - panel.canvas.draw_idle() + if panel is not None: + panel.canvas.draw_idle() + + run_box = ttk.LabelFrame(side_tools, text="Clustering") + run_box.grid(row=0, column=0, sticky="ew", pady=(0, 10)) + run_box.columnconfigure(1, weight=1) + + run_btn = ttk.Button( + run_box, + text="Run Clusters", + command=lambda: app.cluster_playlists_dialog(params["method"]), + ) + run_btn.grid(row=0, column=0, padx=5, pady=5, sticky="ew") + + status = ttk.Label(run_box) + status.grid(row=0, column=1, padx=5, pady=5, sticky="w") playlist_btn = ttk.Button( - side_tools, text="Current Playlists", command=panel.show_current_playlists + side_tools, + text="Current Playlists", + command=lambda: panel and panel.show_current_playlists(), + state="normal" if panel is not None else "disabled", ) - playlist_btn.grid(row=0, column=0, sticky="ew", pady=(0, 10)) + playlist_btn.grid(row=1, column=0, sticky="ew", pady=(0, 10)) cluster_box = ttk.LabelFrame(side_tools, text="Cluster Loader") - cluster_box.grid(row=1, column=0, sticky="nsew", pady=(0, 10)) + cluster_box.grid(row=2, column=0, sticky="nsew", pady=(0, 10)) cluster_box.columnconfigure(1, weight=1) ttk.Label(cluster_box, text="Pick cluster:").grid( row=0, column=0, sticky="w", padx=5, pady=(5, 2) ) - panel.cluster_select_var = tk.StringVar() - panel.cluster_combo = ttk.Combobox( + cluster_select_var = tk.StringVar() + cluster_combo = ttk.Combobox( cluster_box, - textvariable=panel.cluster_select_var, - state="readonly", + textvariable=cluster_select_var, + state="readonly" if panel is not None else "disabled", width=10, ) - panel.cluster_combo.grid(row=0, column=1, sticky="ew", padx=5, pady=(5, 2)) + cluster_combo.grid(row=0, column=1, sticky="ew", padx=5, pady=(5, 2)) ttk.Label(cluster_box, text="Or enter #:").grid( row=1, column=0, sticky="w", padx=5, pady=(0, 2) ) - panel.manual_cluster_var = tk.StringVar() + manual_cluster_var = tk.StringVar() manual_entry = ttk.Entry( - cluster_box, textvariable=panel.manual_cluster_var, width=10 + cluster_box, + textvariable=manual_cluster_var, + width=10, + state="normal" if panel is not None else "disabled", ) manual_entry.grid(row=1, column=1, sticky="ew", padx=5, pady=(0, 2)) def _load_cluster(): - choice = panel.manual_cluster_var.get().strip() or panel.cluster_select_var.get().strip() + if panel is None: + return + choice = manual_cluster_var.get().strip() or cluster_select_var.get().strip() if not choice: messagebox.showinfo("Clusters", "Choose a cluster first.") return @@ -1024,17 +1032,21 @@ def _load_cluster(): return panel.load_cluster(cid) - ttk.Button(cluster_box, text="Load Cluster", command=_load_cluster).grid( - row=2, column=0, columnspan=2, sticky="ew", padx=5, pady=(0, 5) + load_btn = ttk.Button( + cluster_box, + text="Load Cluster", + command=_load_cluster, + state="normal" if panel is not None else "disabled", ) + load_btn.grid(row=2, column=0, columnspan=2, sticky="ew", padx=5, pady=(0, 5)) - panel.temp_status_var = tk.StringVar(value="Clusters available: 0") - ttk.Label(cluster_box, textvariable=panel.temp_status_var).grid( + temp_status_var = tk.StringVar(value="Clusters available: 0") + ttk.Label(cluster_box, textvariable=temp_status_var).grid( row=3, column=0, columnspan=2, sticky="w", padx=5, pady=(0, 5) ) temp_box = ttk.LabelFrame(side_tools, text="Temporary Playlist") - temp_box.grid(row=2, column=0, sticky="nsew") + temp_box.grid(row=3, column=0, sticky="nsew") temp_box.columnconfigure(0, weight=1) temp_box.rowconfigure(0, weight=1) @@ -1043,29 +1055,64 @@ def _load_cluster(): list_frame.columnconfigure(0, weight=1) temp_scroll = ttk.Scrollbar(list_frame, orient="vertical") - panel.temp_listbox = tk.Listbox( - list_frame, height=12, selectmode="extended", yscrollcommand=temp_scroll.set + temp_listbox = tk.Listbox( + list_frame, + height=12, + selectmode="extended", + yscrollcommand=temp_scroll.set, + state="normal" if panel is not None else "disabled", ) - panel.temp_listbox.pack(side="left", fill="both", expand=True) - temp_scroll.config(command=panel.temp_listbox.yview) + temp_listbox.pack(side="left", fill="both", expand=True) + temp_scroll.config(command=temp_listbox.yview) temp_scroll.pack(side="right", fill="y") - ttk.Button(temp_box, text="Show All", command=panel.highlight_temp_playlist).grid( - row=1, column=0, sticky="ew", padx=5, pady=(0, 5) - ) + ttk.Button( + temp_box, + text="Show All", + command=lambda: panel and panel.highlight_temp_playlist(), + state="normal" if panel is not None else "disabled", + ).grid(row=1, column=0, sticky="ew", padx=5, pady=(0, 5)) ttk.Button( - temp_box, text="Add Highlighted Songs", command=panel.add_highlight_to_temp + temp_box, + text="Add Highlighted Songs", + command=lambda: panel and panel.add_highlight_to_temp(), + state="normal" if panel is not None else "disabled", ).grid(row=2, column=0, sticky="ew", padx=5, pady=(0, 5)) - panel.temp_remove_btn = ttk.Button( - temp_box, text="Remove Selected", command=panel.remove_selected_from_temp + temp_remove_btn = ttk.Button( + temp_box, + text="Remove Selected", + command=lambda: panel and panel.remove_selected_from_temp(), + state="normal" if panel is not None else "disabled", ) - panel.temp_remove_btn.grid(row=3, column=0, sticky="ew", padx=5, pady=(0, 5)) + temp_remove_btn.grid(row=3, column=0, sticky="ew", padx=5, pady=(0, 5)) - ttk.Button(temp_box, text="Create Playlist", command=panel.create_temp_playlist).grid( - row=4, column=0, sticky="ew", padx=5, pady=(0, 5) - ) + ttk.Button( + temp_box, + text="Create Playlist", + command=lambda: panel and panel.create_temp_playlist(), + state="normal" if panel is not None else "disabled", + ).grid(row=4, column=0, sticky="ew", padx=5, pady=(0, 5)) + + if getattr(app, "cluster_generation_running", False): + status.config(text="Clustering in progress…") + run_btn.state(["disabled"]) + elif panel is None: + status.config(text="Run clustering once first") + else: + status.config(text="Clusters loaded") + + if panel is not None: + panel.cluster_select_var = cluster_select_var + panel.cluster_combo = cluster_combo + panel.manual_cluster_var = manual_cluster_var + panel.temp_status_var = temp_status_var + panel.temp_listbox = temp_listbox + panel.temp_remove_btn = temp_remove_btn + + if panel is None: + return frame ttk.Separator(container, orient="horizontal").grid(row=1, column=0, sticky="ew") From c968abc8b18c5dbc702d293303cfdcf919ec7bbf Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 15 Dec 2025 20:35:16 -0500 Subject: [PATCH 064/606] Fix cluster graph sizing and add control banner --- cluster_graph_panel.py | 63 ++++++++++++++++++++++++++++++------------ main_gui.py | 18 ++++-------- 2 files changed, 51 insertions(+), 30 deletions(-) diff --git a/cluster_graph_panel.py b/cluster_graph_panel.py index 2d1f565..c5595f1 100644 --- a/cluster_graph_panel.py +++ b/cluster_graph_panel.py @@ -52,10 +52,16 @@ def __init__( self.cluster_manager = cluster_manager self.algo_key = algo_key + self._geometry_ready = False + self._pending_draw_labels: np.ndarray | None = None + self._configure_binding = None + self._map_binding = None + fig = Figure(figsize=(5, 5)) self.ax = fig.add_subplot(111) self.canvas = FigureCanvasTkAgg(fig, master=self) self.canvas.get_tk_widget().pack(fill="both", expand=True) + self._bind_first_geometry_event() self.scatter = None self._loading_var = tk.StringVar(value="Running clustering …") @@ -418,25 +424,48 @@ def recluster(self, params: dict): def _render_clusters(self, labels) -> None: """Finalize layout before drawing clusters to avoid oversized renders.""" + def _queue_draw() -> None: + self._pending_draw_labels = labels + self._maybe_draw_after_geometry() - def _do_draw() -> None: - layout_owner = self.master if self.master is not None else self - try: - layout_owner.update_idletasks() - except Exception: - pass - try: - self.winfo_toplevel().update_idletasks() - except Exception: - pass - try: - self.canvas.resize_event() - except Exception: - pass - self._draw_clusters(labels) + self.after_idle(_queue_draw) + + def _bind_first_geometry_event(self) -> None: + widget = self.canvas.get_tk_widget() + + def _on_geometry(event): + if self._geometry_ready: + return + if event.width <= 1 or event.height <= 1: + return + self._geometry_ready = True + if self._configure_binding: + widget.unbind("", self._configure_binding) + if self._map_binding: + widget.unbind("", self._map_binding) + self._maybe_draw_after_geometry() + + self._configure_binding = widget.bind("", _on_geometry) + self._map_binding = widget.bind("", _on_geometry, add="+") + + def _maybe_draw_after_geometry(self) -> None: + if not self._geometry_ready or self._pending_draw_labels is None: + return + + labels = self._pending_draw_labels + self._pending_draw_labels = None + + try: + self.winfo_toplevel().update_idletasks() + except Exception: + pass + + try: + self.canvas.resize_event() + except Exception: + pass - # Defer to idle to ensure geometry managers finish placing the widgets - self.after_idle(_do_draw) + self._draw_clusters(labels) def _sync_controls(self): """Enable/disable controls based on clustering readiness.""" diff --git a/main_gui.py b/main_gui.py index bf89f74..b686c1c 100644 --- a/main_gui.py +++ b/main_gui.py @@ -955,31 +955,23 @@ def generate(): ) placeholder.grid(row=0, column=0, sticky="nsew") - # Let Tk settle sizes before showing the side panel so the graph owns space - container.update_idletasks() - side_tools = ttk.Frame(container) side_tools.grid(row=0, column=1, rowspan=3, sticky="ns", padx=(10, 0)) side_tools.columnconfigure(0, weight=1) side_tools.rowconfigure(3, weight=1) - # Once the side panel is present, refresh geometry and redraw the graph - container.update_idletasks() - if panel is not None: - panel.canvas.draw_idle() - - run_box = ttk.LabelFrame(side_tools, text="Clustering") - run_box.grid(row=0, column=0, sticky="ew", pady=(0, 10)) - run_box.columnconfigure(1, weight=1) + control_banner = ttk.Frame(side_tools) + control_banner.grid(row=0, column=0, sticky="ew", pady=(0, 10)) + control_banner.columnconfigure(1, weight=1) run_btn = ttk.Button( - run_box, + control_banner, text="Run Clusters", command=lambda: app.cluster_playlists_dialog(params["method"]), ) run_btn.grid(row=0, column=0, padx=5, pady=5, sticky="ew") - status = ttk.Label(run_box) + status = ttk.Label(control_banner) status.grid(row=0, column=1, padx=5, pady=5, sticky="w") playlist_btn = ttk.Button( From a708ee3eaee31d024ac4b180e8ff1973e15220e8 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 15 Dec 2025 20:40:50 -0500 Subject: [PATCH 065/606] Fix canvas geometry binding to preserve resize --- cluster_graph_panel.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cluster_graph_panel.py b/cluster_graph_panel.py index c5595f1..16ce8b0 100644 --- a/cluster_graph_panel.py +++ b/cluster_graph_panel.py @@ -436,7 +436,9 @@ def _bind_first_geometry_event(self) -> None: def _on_geometry(event): if self._geometry_ready: return - if event.width <= 1 or event.height <= 1: + w = getattr(event, "width", widget.winfo_width()) + h = getattr(event, "height", widget.winfo_height()) + if w <= 1 or h <= 1: return self._geometry_ready = True if self._configure_binding: @@ -445,7 +447,7 @@ def _on_geometry(event): widget.unbind("", self._map_binding) self._maybe_draw_after_geometry() - self._configure_binding = widget.bind("", _on_geometry) + self._configure_binding = widget.bind("", _on_geometry, add="+") self._map_binding = widget.bind("", _on_geometry, add="+") def _maybe_draw_after_geometry(self) -> None: From 5e12c21783a0161742b8ea200aafdbf6bd2c631e Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 15 Dec 2025 20:46:32 -0500 Subject: [PATCH 066/606] Schedule final redraw after clusters render --- cluster_graph_panel.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/cluster_graph_panel.py b/cluster_graph_panel.py index 16ce8b0..1471603 100644 --- a/cluster_graph_panel.py +++ b/cluster_graph_panel.py @@ -469,6 +469,25 @@ def _maybe_draw_after_geometry(self) -> None: self._draw_clusters(labels) + def final_redraw(): + try: + self.winfo_toplevel().update_idletasks() + except Exception: + pass + + try: + self.canvas.resize_event() + except Exception: + pass + + try: + self.canvas.draw_idle() + except Exception: + pass + + self.after_idle(final_redraw) + self.after(30, final_redraw) + def _sync_controls(self): """Enable/disable controls based on clustering readiness.""" state = "normal" if self._clusters_ready else "disabled" From 6d2de44e5b873da5daa90d192f7a0c7616c50985 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 15 Dec 2025 20:51:40 -0500 Subject: [PATCH 067/606] Fix cluster dropdown initialization --- main_gui.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/main_gui.py b/main_gui.py index b686c1c..9017b69 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1102,6 +1102,8 @@ def _load_cluster(): panel.temp_status_var = temp_status_var panel.temp_listbox = temp_listbox panel.temp_remove_btn = temp_remove_btn + panel.refresh_control_states() + panel._refresh_cluster_options() if panel is None: return frame From 9136f38a0ad60be11f5b8b216a11c2e11923b4ec Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 15 Dec 2025 20:57:13 -0500 Subject: [PATCH 068/606] Fix cluster dropdown and stabilize controls --- cluster_graph_panel.py | 8 +++++++- main_gui.py | 4 ++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/cluster_graph_panel.py b/cluster_graph_panel.py index 1471603..0dbae97 100644 --- a/cluster_graph_panel.py +++ b/cluster_graph_panel.py @@ -395,6 +395,7 @@ def _draw_clusters(self, labels): self.scatter.set_offsets(self.X2) self.scatter.set_color(point_colors) self.canvas.draw_idle() + self._refresh_cluster_options() def _on_clusters_ready(self, labels, params: dict): duration_ms = None @@ -508,7 +509,12 @@ def _refresh_cluster_options(self): return clusters = sorted({int(l) for l in set(self.labels) if l >= 0}) if self.cluster_combo is not None: - self.cluster_combo.configure(values=[str(c) for c in clusters]) + values = [str(c) for c in clusters] + self.cluster_combo.configure(values=values) + if self.cluster_select_var is not None: + current = self.cluster_select_var.get() + if current not in values and values: + self.cluster_select_var.set(values[0]) if self.temp_status_var is not None: self.temp_status_var.set(f"Clusters available: {len(clusters)}") diff --git a/main_gui.py b/main_gui.py index 9017b69..6faacac 100644 --- a/main_gui.py +++ b/main_gui.py @@ -960,6 +960,10 @@ def generate(): side_tools.columnconfigure(0, weight=1) side_tools.rowconfigure(3, weight=1) + # Keep the control column stable while the graph updates + side_tools.update_idletasks() + container.columnconfigure(1, minsize=side_tools.winfo_reqwidth()) + control_banner = ttk.Frame(side_tools) control_banner.grid(row=0, column=0, sticky="ew", pady=(0, 10)) control_banner.columnconfigure(1, weight=1) From c1dff0c9f96f2432e2bd8602280752d1a3fb73e5 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 15 Dec 2025 21:24:11 -0500 Subject: [PATCH 069/606] Defer cluster graph until data ready --- main_gui.py | 75 ++++++++++++++++++++++++++++------------------------- 1 file changed, 39 insertions(+), 36 deletions(-) diff --git a/main_gui.py b/main_gui.py index 6faacac..9fbec77 100644 --- a/main_gui.py +++ b/main_gui.py @@ -137,6 +137,7 @@ def create_panel_for_plugin(app, name: str, parent: tk.Widget) -> ttk.Frame: cluster_data = getattr(app, "cluster_data", None) cluster_cfg = getattr(app, "cluster_params", None) cluster_manager = getattr(app, "cluster_manager", None) + cluster_running = getattr(app, "cluster_generation_running", False) if cluster_data is None: tracks = features = None else: @@ -918,6 +919,25 @@ def generate(): ttk.Label(frame, text=f"{name} panel coming soon…").pack(padx=10, pady=10) return frame + has_cluster_data = cluster_manager is not None and cluster_data is not None + + if cluster_running or not has_cluster_data: + banner = ttk.Frame(frame) + banner.pack(fill="both", expand=True, padx=10, pady=10) + banner.columnconfigure(1, weight=1) + + run_btn = ttk.Button( + banner, + text="Run Clusters", + command=lambda: app.cluster_playlists_dialog(params["method"]), + state="disabled" if cluster_running else "normal", + ) + run_btn.grid(row=0, column=0, padx=5, pady=5, sticky="ew") + + msg = "Clustering in progress…" if cluster_running else "Run clustering once first" + ttk.Label(banner, text=msg).grid(row=0, column=1, padx=5, pady=5, sticky="w") + return frame + # Container ensures the graph resizes while keeping controls visible container = ttk.Frame(frame) container.pack(fill="both", expand=True) @@ -931,29 +951,21 @@ def generate(): graph_area.columnconfigure(0, weight=1) panel: ClusterGraphPanel | None = None - if cluster_manager is not None: - X, X2 = cluster_manager.get_projection() - - panel = ClusterGraphPanel( - graph_area, - tracks, - X, - X2, - cluster_func=km_func, - cluster_params=params, - cluster_manager=cluster_manager, - algo_key=algo_key, - library_path=app.library_path, - log_callback=app._log, - ) - panel.grid(row=0, column=0, sticky="nsew", pady=(0, 5)) - else: - placeholder = ttk.Label( - graph_area, - text="Run clustering once to view the interactive graph.", - anchor="center", - ) - placeholder.grid(row=0, column=0, sticky="nsew") + X, X2 = cluster_manager.get_projection() + + panel = ClusterGraphPanel( + graph_area, + tracks, + X, + X2, + cluster_func=km_func, + cluster_params=params, + cluster_manager=cluster_manager, + algo_key=algo_key, + library_path=app.library_path, + log_callback=app._log, + ) + panel.grid(row=0, column=0, sticky="nsew", pady=(0, 5)) side_tools = ttk.Frame(container) side_tools.grid(row=0, column=1, rowspan=3, sticky="ns", padx=(10, 0)) @@ -1091,13 +1103,7 @@ def _load_cluster(): state="normal" if panel is not None else "disabled", ).grid(row=4, column=0, sticky="ew", padx=5, pady=(0, 5)) - if getattr(app, "cluster_generation_running", False): - status.config(text="Clustering in progress…") - run_btn.state(["disabled"]) - elif panel is None: - status.config(text="Run clustering once first") - else: - status.config(text="Clusters loaded") + status.config(text="Clusters loaded") if panel is not None: panel.cluster_select_var = cluster_select_var @@ -1154,12 +1160,7 @@ def _auto_create_all(): engine = panel.cluster_params.get("engine", "librosa") if not params: return - app.show_log_tab() - threading.Thread( - target=app._run_cluster_generation, - args=(app.library_path, method, params, engine), - daemon=True, - ).start() + app._start_cluster_playlists(method, params, engine, None) auto_btn = ttk.Button(btn_frame, text="Auto-Create", command=_auto_create_all) auto_btn.pack(side="left", padx=(5, 0)) @@ -1329,6 +1330,7 @@ def __init__(self): # Cached tracks and feature vectors for interactive clustering self.cluster_data = None self.cluster_manager = None + self.cluster_generation_running = False self.folder_filter = {"include": [], "exclude": []} # Shared audio feature/analysis engine selection @@ -2073,6 +2075,7 @@ def select_library(self): # Clear any cached clustering data when switching libraries self.cluster_data = None self.cluster_manager = None + self.cluster_generation_running = False self.plugin_views.clear() self.active_plugin = None if hasattr(self, "scan_btn"): From 6995e97e1100211ccc8b464ef318f6d0e6079863 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 15 Dec 2025 21:38:23 -0500 Subject: [PATCH 070/606] Revert "Defer cluster graph until data ready" --- main_gui.py | 75 +++++++++++++++++++++++++---------------------------- 1 file changed, 36 insertions(+), 39 deletions(-) diff --git a/main_gui.py b/main_gui.py index 9fbec77..6faacac 100644 --- a/main_gui.py +++ b/main_gui.py @@ -137,7 +137,6 @@ def create_panel_for_plugin(app, name: str, parent: tk.Widget) -> ttk.Frame: cluster_data = getattr(app, "cluster_data", None) cluster_cfg = getattr(app, "cluster_params", None) cluster_manager = getattr(app, "cluster_manager", None) - cluster_running = getattr(app, "cluster_generation_running", False) if cluster_data is None: tracks = features = None else: @@ -919,25 +918,6 @@ def generate(): ttk.Label(frame, text=f"{name} panel coming soon…").pack(padx=10, pady=10) return frame - has_cluster_data = cluster_manager is not None and cluster_data is not None - - if cluster_running or not has_cluster_data: - banner = ttk.Frame(frame) - banner.pack(fill="both", expand=True, padx=10, pady=10) - banner.columnconfigure(1, weight=1) - - run_btn = ttk.Button( - banner, - text="Run Clusters", - command=lambda: app.cluster_playlists_dialog(params["method"]), - state="disabled" if cluster_running else "normal", - ) - run_btn.grid(row=0, column=0, padx=5, pady=5, sticky="ew") - - msg = "Clustering in progress…" if cluster_running else "Run clustering once first" - ttk.Label(banner, text=msg).grid(row=0, column=1, padx=5, pady=5, sticky="w") - return frame - # Container ensures the graph resizes while keeping controls visible container = ttk.Frame(frame) container.pack(fill="both", expand=True) @@ -951,21 +931,29 @@ def generate(): graph_area.columnconfigure(0, weight=1) panel: ClusterGraphPanel | None = None - X, X2 = cluster_manager.get_projection() - - panel = ClusterGraphPanel( - graph_area, - tracks, - X, - X2, - cluster_func=km_func, - cluster_params=params, - cluster_manager=cluster_manager, - algo_key=algo_key, - library_path=app.library_path, - log_callback=app._log, - ) - panel.grid(row=0, column=0, sticky="nsew", pady=(0, 5)) + if cluster_manager is not None: + X, X2 = cluster_manager.get_projection() + + panel = ClusterGraphPanel( + graph_area, + tracks, + X, + X2, + cluster_func=km_func, + cluster_params=params, + cluster_manager=cluster_manager, + algo_key=algo_key, + library_path=app.library_path, + log_callback=app._log, + ) + panel.grid(row=0, column=0, sticky="nsew", pady=(0, 5)) + else: + placeholder = ttk.Label( + graph_area, + text="Run clustering once to view the interactive graph.", + anchor="center", + ) + placeholder.grid(row=0, column=0, sticky="nsew") side_tools = ttk.Frame(container) side_tools.grid(row=0, column=1, rowspan=3, sticky="ns", padx=(10, 0)) @@ -1103,7 +1091,13 @@ def _load_cluster(): state="normal" if panel is not None else "disabled", ).grid(row=4, column=0, sticky="ew", padx=5, pady=(0, 5)) - status.config(text="Clusters loaded") + if getattr(app, "cluster_generation_running", False): + status.config(text="Clustering in progress…") + run_btn.state(["disabled"]) + elif panel is None: + status.config(text="Run clustering once first") + else: + status.config(text="Clusters loaded") if panel is not None: panel.cluster_select_var = cluster_select_var @@ -1160,7 +1154,12 @@ def _auto_create_all(): engine = panel.cluster_params.get("engine", "librosa") if not params: return - app._start_cluster_playlists(method, params, engine, None) + app.show_log_tab() + threading.Thread( + target=app._run_cluster_generation, + args=(app.library_path, method, params, engine), + daemon=True, + ).start() auto_btn = ttk.Button(btn_frame, text="Auto-Create", command=_auto_create_all) auto_btn.pack(side="left", padx=(5, 0)) @@ -1330,7 +1329,6 @@ def __init__(self): # Cached tracks and feature vectors for interactive clustering self.cluster_data = None self.cluster_manager = None - self.cluster_generation_running = False self.folder_filter = {"include": [], "exclude": []} # Shared audio feature/analysis engine selection @@ -2075,7 +2073,6 @@ def select_library(self): # Clear any cached clustering data when switching libraries self.cluster_data = None self.cluster_manager = None - self.cluster_generation_running = False self.plugin_views.clear() self.active_plugin = None if hasattr(self, "scan_btn"): From 5f105c63684f65086c8ec4c22095481de43fdc98 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 15 Dec 2025 21:58:23 -0500 Subject: [PATCH 071/606] Guard cluster graph rendering during generation --- main_gui.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/main_gui.py b/main_gui.py index 6faacac..3d25e8b 100644 --- a/main_gui.py +++ b/main_gui.py @@ -931,7 +931,15 @@ def generate(): graph_area.columnconfigure(0, weight=1) panel: ClusterGraphPanel | None = None - if cluster_manager is not None: + placeholder_text: str | None = None + cluster_generation_running = getattr(app, "cluster_generation_running", False) + cluster_ready = cluster_manager is not None and cluster_data is not None + + if cluster_generation_running: + placeholder_text = "Clustering in progress…" + elif not cluster_ready: + placeholder_text = "Run clustering once first" + else: X, X2 = cluster_manager.get_projection() panel = ClusterGraphPanel( @@ -947,12 +955,9 @@ def generate(): log_callback=app._log, ) panel.grid(row=0, column=0, sticky="nsew", pady=(0, 5)) - else: - placeholder = ttk.Label( - graph_area, - text="Run clustering once to view the interactive graph.", - anchor="center", - ) + + if placeholder_text: + placeholder = ttk.Label(graph_area, text=placeholder_text, anchor="center") placeholder.grid(row=0, column=0, sticky="nsew") side_tools = ttk.Frame(container) @@ -1091,10 +1096,10 @@ def _load_cluster(): state="normal" if panel is not None else "disabled", ).grid(row=4, column=0, sticky="ew", padx=5, pady=(0, 5)) - if getattr(app, "cluster_generation_running", False): + if cluster_generation_running: status.config(text="Clustering in progress…") run_btn.state(["disabled"]) - elif panel is None: + elif not cluster_ready: status.config(text="Run clustering once first") else: status.config(text="Clusters loaded") @@ -1329,6 +1334,7 @@ def __init__(self): # Cached tracks and feature vectors for interactive clustering self.cluster_data = None self.cluster_manager = None + self.cluster_generation_running = False self.folder_filter = {"include": [], "exclude": []} # Shared audio feature/analysis engine selection From 6362cad1a7096e8fc72155b1abbe9d5a1ca2ca2c Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 15 Dec 2025 22:13:27 -0500 Subject: [PATCH 072/606] Keep clustering controls alive during generation --- main_gui.py | 240 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 146 insertions(+), 94 deletions(-) diff --git a/main_gui.py b/main_gui.py index 3d25e8b..6d0bae1 100644 --- a/main_gui.py +++ b/main_gui.py @@ -931,34 +931,6 @@ def generate(): graph_area.columnconfigure(0, weight=1) panel: ClusterGraphPanel | None = None - placeholder_text: str | None = None - cluster_generation_running = getattr(app, "cluster_generation_running", False) - cluster_ready = cluster_manager is not None and cluster_data is not None - - if cluster_generation_running: - placeholder_text = "Clustering in progress…" - elif not cluster_ready: - placeholder_text = "Run clustering once first" - else: - X, X2 = cluster_manager.get_projection() - - panel = ClusterGraphPanel( - graph_area, - tracks, - X, - X2, - cluster_func=km_func, - cluster_params=params, - cluster_manager=cluster_manager, - algo_key=algo_key, - library_path=app.library_path, - log_callback=app._log, - ) - panel.grid(row=0, column=0, sticky="nsew", pady=(0, 5)) - - if placeholder_text: - placeholder = ttk.Label(graph_area, text=placeholder_text, anchor="center") - placeholder.grid(row=0, column=0, sticky="nsew") side_tools = ttk.Frame(container) side_tools.grid(row=0, column=1, rowspan=3, sticky="ns", padx=(10, 0)) @@ -987,7 +959,6 @@ def generate(): side_tools, text="Current Playlists", command=lambda: panel and panel.show_current_playlists(), - state="normal" if panel is not None else "disabled", ) playlist_btn.grid(row=1, column=0, sticky="ew", pady=(0, 10)) @@ -1002,7 +973,6 @@ def generate(): cluster_combo = ttk.Combobox( cluster_box, textvariable=cluster_select_var, - state="readonly" if panel is not None else "disabled", width=10, ) cluster_combo.grid(row=0, column=1, sticky="ew", padx=5, pady=(5, 2)) @@ -1015,7 +985,6 @@ def generate(): cluster_box, textvariable=manual_cluster_var, width=10, - state="normal" if panel is not None else "disabled", ) manual_entry.grid(row=1, column=1, sticky="ew", padx=5, pady=(0, 2)) @@ -1037,7 +1006,6 @@ def _load_cluster(): cluster_box, text="Load Cluster", command=_load_cluster, - state="normal" if panel is not None else "disabled", ) load_btn.grid(row=2, column=0, columnspan=2, sticky="ew", padx=5, pady=(0, 5)) @@ -1061,95 +1029,71 @@ def _load_cluster(): height=12, selectmode="extended", yscrollcommand=temp_scroll.set, - state="normal" if panel is not None else "disabled", ) temp_listbox.pack(side="left", fill="both", expand=True) temp_scroll.config(command=temp_listbox.yview) temp_scroll.pack(side="right", fill="y") - ttk.Button( + show_all_btn = ttk.Button( temp_box, text="Show All", command=lambda: panel and panel.highlight_temp_playlist(), - state="normal" if panel is not None else "disabled", - ).grid(row=1, column=0, sticky="ew", padx=5, pady=(0, 5)) + ) + show_all_btn.grid(row=1, column=0, sticky="ew", padx=5, pady=(0, 5)) - ttk.Button( + add_highlight_btn = ttk.Button( temp_box, text="Add Highlighted Songs", command=lambda: panel and panel.add_highlight_to_temp(), - state="normal" if panel is not None else "disabled", - ).grid(row=2, column=0, sticky="ew", padx=5, pady=(0, 5)) + ) + add_highlight_btn.grid(row=2, column=0, sticky="ew", padx=5, pady=(0, 5)) temp_remove_btn = ttk.Button( temp_box, text="Remove Selected", command=lambda: panel and panel.remove_selected_from_temp(), - state="normal" if panel is not None else "disabled", ) temp_remove_btn.grid(row=3, column=0, sticky="ew", padx=5, pady=(0, 5)) - ttk.Button( + create_playlist_btn = ttk.Button( temp_box, text="Create Playlist", command=lambda: panel and panel.create_temp_playlist(), - state="normal" if panel is not None else "disabled", - ).grid(row=4, column=0, sticky="ew", padx=5, pady=(0, 5)) - - if cluster_generation_running: - status.config(text="Clustering in progress…") - run_btn.state(["disabled"]) - elif not cluster_ready: - status.config(text="Run clustering once first") - else: - status.config(text="Clusters loaded") - - if panel is not None: - panel.cluster_select_var = cluster_select_var - panel.cluster_combo = cluster_combo - panel.manual_cluster_var = manual_cluster_var - panel.temp_status_var = temp_status_var - panel.temp_listbox = temp_listbox - panel.temp_remove_btn = temp_remove_btn - panel.refresh_control_states() - panel._refresh_cluster_options() - - if panel is None: - return frame + ) + create_playlist_btn.grid(row=4, column=0, sticky="ew", padx=5, pady=(0, 5)) ttk.Separator(container, orient="horizontal").grid(row=1, column=0, sticky="ew") btn_frame = ttk.Frame(container) btn_frame.grid(row=2, column=0, sticky="ew", pady=5) - panel.lasso_var = tk.BooleanVar(value=False) + lasso_var = tk.BooleanVar(value=False) lasso_btn = ttk.Checkbutton( btn_frame, text="Lasso Mode", - variable=panel.lasso_var, - command=panel.toggle_lasso, + variable=lasso_var, + command=lambda: panel and panel.toggle_lasso(), ) lasso_btn.pack(side="left") - panel.lasso_btn = lasso_btn - panel.ok_btn = ttk.Button( + ok_btn = ttk.Button( btn_frame, text="OK", - command=panel.finalize_lasso, - state="disabled", + command=lambda: panel and panel.finalize_lasso(), ) - panel.ok_btn.pack(side="left", padx=(5, 0)) + ok_btn.pack(side="left", padx=(5, 0)) - panel.gen_btn = ttk.Button( + gen_btn = ttk.Button( btn_frame, text="Generate Playlist", - command=panel.create_playlist, - state="disabled", + command=lambda: panel and panel.create_playlist(), ) - panel.gen_btn.pack(side="left", padx=(5, 0)) + gen_btn.pack(side="left", padx=(5, 0)) def _auto_create_all(): + if panel is None or not getattr(panel, "cluster_params", None): + return method = panel.cluster_params.get("method") params = { k: v @@ -1169,28 +1113,131 @@ def _auto_create_all(): auto_btn = ttk.Button(btn_frame, text="Auto-Create", command=_auto_create_all) auto_btn.pack(side="left", padx=(5, 0)) + redo_btn: ttk.Button | None = None if name == "Interactive – HDBSCAN": redo_btn = ttk.Button( - btn_frame, text="Redo Values", command=panel.open_param_dialog + btn_frame, text="Redo Values", command=lambda: panel and panel.open_param_dialog() ) redo_btn.pack(side="left", padx=(5, 0)) - # ─── Hover Metadata Panel ──────────────────────────────────────────── - hover_panel = ttk.Frame(panel, relief="solid", borderwidth=1) - art_lbl = tk.Label(hover_panel) - art_lbl.pack(side="left") - text_frame = ttk.Frame(hover_panel) - text_frame.pack(side="left", padx=5) - title_lbl = ttk.Label(text_frame, font=("TkDefaultFont", 10, "bold")) - title_lbl.pack(anchor="w") - artist_lbl = ttk.Label(text_frame, font=("TkDefaultFont", 8)) - artist_lbl.pack(anchor="w") - hover_panel.place(relx=1.0, rely=1.0, anchor="se", x=-10, y=-10) - hover_panel.place_forget() - - panel.setup_hover(hover_panel, art_lbl, title_lbl, artist_lbl) - panel._refresh_cluster_options() - panel.refresh_control_states() + def _update_control_states(): + running = getattr(app, "cluster_generation_running", False) + ready = panel is not None + + for widget in ( + playlist_btn, + cluster_combo, + manual_entry, + load_btn, + temp_listbox, + show_all_btn, + add_highlight_btn, + temp_remove_btn, + create_playlist_btn, + lasso_btn, + ok_btn, + gen_btn, + auto_btn, + ): + widget_state = "normal" if ready and not running else "disabled" + widget.config(state=widget_state) + + if redo_btn is not None: + redo_btn_state = "normal" if ready and not running else "disabled" + redo_btn.config(state=redo_btn_state) + + run_btn.state(["disabled"] if running else ["!disabled"]) + + if running: + status.config(text="Clustering in progress…") + elif not ready: + status.config(text="Run clustering once first") + else: + status.config(text="Clusters loaded") + + def refresh_graph_area(): + nonlocal panel, cluster_manager, cluster_data + + cluster_data = getattr(app, "cluster_data", None) + cluster_manager = getattr(app, "cluster_manager", None) + if cluster_data is None: + tracks_local = features_local = None + else: + tracks_local, features_local = cluster_data + if cluster_manager is None or getattr(cluster_manager, "tracks", None) != tracks_local: + app.cluster_manager = ClusterComputationManager(tracks_local, features_local, app._log) + cluster_manager = app.cluster_manager + + cluster_generation_running = getattr(app, "cluster_generation_running", False) + cluster_ready = cluster_manager is not None and cluster_data is not None + + for child in graph_area.winfo_children(): + child.destroy() + + panel = None + placeholder_text: str | None = None + + if cluster_generation_running: + placeholder_text = "Clustering in progress…" + elif not cluster_ready: + placeholder_text = "Run clustering once first" + else: + X, X2 = cluster_manager.get_projection() + + panel = ClusterGraphPanel( + graph_area, + tracks_local, + X, + X2, + cluster_func=km_func, + cluster_params=params, + cluster_manager=cluster_manager, + algo_key=algo_key, + library_path=app.library_path, + log_callback=app._log, + ) + panel.grid(row=0, column=0, sticky="nsew", pady=(0, 5)) + + if placeholder_text: + overlay = ttk.Label(graph_area, text=placeholder_text, anchor="center") + overlay.grid(row=0, column=0, sticky="nsew") + + if panel is not None: + panel.cluster_select_var = cluster_select_var + panel.cluster_combo = cluster_combo + panel.manual_cluster_var = manual_cluster_var + panel.temp_status_var = temp_status_var + panel.temp_listbox = temp_listbox + panel.temp_remove_btn = temp_remove_btn + panel.lasso_var = lasso_var + panel.lasso_btn = lasso_btn + panel.ok_btn = ok_btn + panel.gen_btn = gen_btn + panel.refresh_control_states() + panel._refresh_cluster_options() + + hover_panel = ttk.Frame(panel, relief="solid", borderwidth=1) + art_lbl = tk.Label(hover_panel) + art_lbl.pack(side="left") + text_frame = ttk.Frame(hover_panel) + text_frame.pack(side="left", padx=5) + title_lbl = ttk.Label(text_frame, font=("TkDefaultFont", 10, "bold")) + title_lbl.pack(anchor="w") + artist_lbl = ttk.Label(text_frame, font=("TkDefaultFont", 8)) + artist_lbl.pack(anchor="w") + hover_panel.place(relx=1.0, rely=1.0, anchor="se", x=-10, y=-10) + hover_panel.place_forget() + + panel.setup_hover(hover_panel, art_lbl, title_lbl, artist_lbl) + + _update_control_states() + + refresh_graph_area() + + def refresh_cluster_panel(): + refresh_graph_area() + + frame.refresh_cluster_panel = refresh_cluster_panel # type: ignore[attr-defined] return frame @@ -2698,8 +2745,13 @@ def _refresh_plugin_panel(self): except tk.TclError: return if sel in self.plugin_views: + panel = self.plugin_views[sel] + refresh = getattr(panel, "refresh_cluster_panel", None) + if callable(refresh): + refresh() + return try: - self.plugin_views[sel].destroy() + panel.destroy() except Exception: pass self.plugin_views.pop(sel, None) From fdfec9964b223801ac0e526db00a4fe732be50a2 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 15 Dec 2025 22:22:03 -0500 Subject: [PATCH 073/606] Simplify cluster graph refresh --- cluster_graph_panel.py | 28 +++++++++++++++ main_gui.py | 82 ++++++++++++++++++++++++++---------------- 2 files changed, 80 insertions(+), 30 deletions(-) diff --git a/cluster_graph_panel.py b/cluster_graph_panel.py index 0dbae97..20a5ff5 100644 --- a/cluster_graph_panel.py +++ b/cluster_graph_panel.py @@ -51,6 +51,7 @@ def __init__( self.log = log_callback self.cluster_manager = cluster_manager self.algo_key = algo_key + self._last_size: tuple[int, int] | None = None self._geometry_ready = False self._pending_draw_labels: np.ndarray | None = None @@ -489,6 +490,33 @@ def final_redraw(): self.after_idle(final_redraw) self.after(30, final_redraw) + def on_resize(self, width: int, height: int) -> None: + """Resize the matplotlib canvas to the given dimensions.""" + + if width <= 1 or height <= 1: + return + + size = (width, height) + if size == self._last_size: + return + self._last_size = size + + widget = self.canvas.get_tk_widget() + widget.configure(width=width, height=height) + + fig = self.canvas.figure + dpi = fig.dpi or 72 + fig.set_size_inches(max(width / dpi, 1e-2), max(height / dpi, 1e-2), forward=False) + + try: + self.canvas.resize_event() + except Exception: + pass + try: + self.canvas.draw_idle() + except Exception: + pass + def _sync_controls(self): """Enable/disable controls based on clustering readiness.""" state = "normal" if self._clusters_ready else "disabled" diff --git a/main_gui.py b/main_gui.py index 6d0bae1..ca9b7d2 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1120,9 +1120,11 @@ def _auto_create_all(): ) redo_btn.pack(side="left", padx=(5, 0)) - def _update_control_states(): - running = getattr(app, "cluster_generation_running", False) - ready = panel is not None + placeholder: ttk.Label | None = None + + def _sync_cluster_controls(cluster_ready: bool, running: bool) -> None: + ready = cluster_ready and panel is not None + state = "normal" if ready and not running else "disabled" for widget in ( playlist_btn, @@ -1139,12 +1141,10 @@ def _update_control_states(): gen_btn, auto_btn, ): - widget_state = "normal" if ready and not running else "disabled" - widget.config(state=widget_state) + widget.config(state=state) if redo_btn is not None: - redo_btn_state = "normal" if ready and not running else "disabled" - redo_btn.config(state=redo_btn_state) + redo_btn.config(state=state) run_btn.state(["disabled"] if running else ["!disabled"]) @@ -1155,7 +1155,17 @@ def _update_control_states(): else: status.config(text="Clusters loaded") - def refresh_graph_area(): + def _ensure_placeholder(message: str) -> ttk.Label: + nonlocal placeholder + + if placeholder is None: + placeholder = ttk.Label(graph_area, anchor="center") + placeholder.grid(row=0, column=0, sticky="nsew") + placeholder.configure(text=message) + placeholder.lift() + return placeholder + + def refresh_cluster_panel(): nonlocal panel, cluster_manager, cluster_data cluster_data = getattr(app, "cluster_data", None) @@ -1165,23 +1175,32 @@ def refresh_graph_area(): else: tracks_local, features_local = cluster_data if cluster_manager is None or getattr(cluster_manager, "tracks", None) != tracks_local: - app.cluster_manager = ClusterComputationManager(tracks_local, features_local, app._log) + app.cluster_manager = ClusterComputationManager( + tracks_local, features_local, app._log + ) cluster_manager = app.cluster_manager cluster_generation_running = getattr(app, "cluster_generation_running", False) cluster_ready = cluster_manager is not None and cluster_data is not None - for child in graph_area.winfo_children(): - child.destroy() + if cluster_generation_running: + if panel is not None: + panel.grid_remove() + _ensure_placeholder("Clustering in progress…") + _sync_cluster_controls(False, True) + return - panel = None - placeholder_text: str | None = None + if not cluster_ready: + if panel is not None: + panel.grid_remove() + _ensure_placeholder("Run clustering once first") + _sync_cluster_controls(False, False) + return - if cluster_generation_running: - placeholder_text = "Clustering in progress…" - elif not cluster_ready: - placeholder_text = "Run clustering once first" - else: + if placeholder is not None: + placeholder.grid_remove() + + if panel is None: X, X2 = cluster_manager.get_projection() panel = ClusterGraphPanel( @@ -1196,13 +1215,7 @@ def refresh_graph_area(): library_path=app.library_path, log_callback=app._log, ) - panel.grid(row=0, column=0, sticky="nsew", pady=(0, 5)) - if placeholder_text: - overlay = ttk.Label(graph_area, text=placeholder_text, anchor="center") - overlay.grid(row=0, column=0, sticky="nsew") - - if panel is not None: panel.cluster_select_var = cluster_select_var panel.cluster_combo = cluster_combo panel.manual_cluster_var = manual_cluster_var @@ -1213,8 +1226,6 @@ def refresh_graph_area(): panel.lasso_btn = lasso_btn panel.ok_btn = ok_btn panel.gen_btn = gen_btn - panel.refresh_control_states() - panel._refresh_cluster_options() hover_panel = ttk.Frame(panel, relief="solid", borderwidth=1) art_lbl = tk.Label(hover_panel) @@ -1230,12 +1241,23 @@ def refresh_graph_area(): panel.setup_hover(hover_panel, art_lbl, title_lbl, artist_lbl) - _update_control_states() + def _handle_resize(event): + if panel is not None: + panel.on_resize(event.width, event.height) - refresh_graph_area() + graph_area.bind("", _handle_resize, add="+") + panel.grid(row=0, column=0, sticky="nsew", pady=(0, 5)) + graph_area.after_idle( + lambda: panel.on_resize(graph_area.winfo_width(), graph_area.winfo_height()) + ) + else: + panel.grid() - def refresh_cluster_panel(): - refresh_graph_area() + panel.refresh_control_states() + panel._refresh_cluster_options() + _sync_cluster_controls(True, False) + + refresh_cluster_panel() frame.refresh_cluster_panel = refresh_cluster_panel # type: ignore[attr-defined] From 21576e0f1ee9877d51a6b324c444e4356bcb824a Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 15 Dec 2025 22:31:03 -0500 Subject: [PATCH 074/606] Keep clustering controls visible during graph refresh --- main_gui.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/main_gui.py b/main_gui.py index ca9b7d2..89c508f 100644 --- a/main_gui.py +++ b/main_gui.py @@ -930,6 +930,13 @@ def generate(): graph_area.rowconfigure(0, weight=1) graph_area.columnconfigure(0, weight=1) + # Keep the controls outside the graph container so they remain visible + # while the graph refreshes. + graph_stack = ttk.Frame(graph_area) + graph_stack.grid(row=0, column=0, sticky="nsew") + graph_stack.rowconfigure(0, weight=1) + graph_stack.columnconfigure(0, weight=1) + panel: ClusterGraphPanel | None = None side_tools = ttk.Frame(container) @@ -1159,7 +1166,7 @@ def _ensure_placeholder(message: str) -> ttk.Label: nonlocal placeholder if placeholder is None: - placeholder = ttk.Label(graph_area, anchor="center") + placeholder = ttk.Label(graph_stack, anchor="center") placeholder.grid(row=0, column=0, sticky="nsew") placeholder.configure(text=message) placeholder.lift() @@ -1184,16 +1191,16 @@ def refresh_cluster_panel(): cluster_ready = cluster_manager is not None and cluster_data is not None if cluster_generation_running: - if panel is not None: - panel.grid_remove() _ensure_placeholder("Clustering in progress…") + if panel is not None: + panel.grid() _sync_cluster_controls(False, True) return if not cluster_ready: - if panel is not None: - panel.grid_remove() _ensure_placeholder("Run clustering once first") + if panel is not None: + panel.grid() _sync_cluster_controls(False, False) return @@ -1204,7 +1211,7 @@ def refresh_cluster_panel(): X, X2 = cluster_manager.get_projection() panel = ClusterGraphPanel( - graph_area, + graph_stack, tracks_local, X, X2, @@ -1245,10 +1252,10 @@ def _handle_resize(event): if panel is not None: panel.on_resize(event.width, event.height) - graph_area.bind("", _handle_resize, add="+") + graph_stack.bind("", _handle_resize, add="+") panel.grid(row=0, column=0, sticky="nsew", pady=(0, 5)) - graph_area.after_idle( - lambda: panel.on_resize(graph_area.winfo_width(), graph_area.winfo_height()) + graph_stack.after_idle( + lambda: panel.on_resize(graph_stack.winfo_width(), graph_stack.winfo_height()) ) else: panel.grid() From f9524f63e48cff383467beae477394fb894e8b6a Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 15 Dec 2025 22:39:36 -0500 Subject: [PATCH 075/606] Fix cluster UI refresh after generation completion --- main_gui.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/main_gui.py b/main_gui.py index 89c508f..405d6a9 100644 --- a/main_gui.py +++ b/main_gui.py @@ -2742,15 +2742,15 @@ def _run_cluster_generation( self.cluster_params = {"method": method, "engine": engine, **params} self.cluster_manager = ClusterComputationManager(tracks, feats, self._log) - for name in ("Interactive – KMeans", "Interactive – HDBSCAN"): - if name in self.plugin_views: - try: - self.plugin_views[name].destroy() - except Exception: - pass - self.plugin_views.pop(name, None) - def done(): + for name in ("Interactive – KMeans", "Interactive – HDBSCAN"): + if name in self.plugin_views: + try: + self.plugin_views[name].destroy() + except Exception: + pass + self.plugin_views.pop(name, None) + self.cluster_generation_running = False messagebox.showinfo("Clustered Playlists", "Generation complete") self._refresh_plugin_panel() From e87c535092b1c1e195b86b45e64a5231849d7864 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 15 Dec 2025 22:44:42 -0500 Subject: [PATCH 076/606] Preserve cluster plugin controls after generation --- main_gui.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/main_gui.py b/main_gui.py index 89c508f..cd5fd7c 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1190,6 +1190,10 @@ def refresh_cluster_panel(): cluster_generation_running = getattr(app, "cluster_generation_running", False) cluster_ready = cluster_manager is not None and cluster_data is not None + if panel is not None and cluster_manager is not None and panel.cluster_manager != cluster_manager: + panel.destroy() + panel = None + if cluster_generation_running: _ensure_placeholder("Clustering in progress…") if panel is not None: @@ -1260,6 +1264,10 @@ def _handle_resize(event): else: panel.grid() + # Keep panel state in sync with the latest clustering run + panel.cluster_params = params + panel.cluster_manager = cluster_manager + panel.refresh_control_states() panel._refresh_cluster_options() _sync_cluster_controls(True, False) @@ -2742,14 +2750,6 @@ def _run_cluster_generation( self.cluster_params = {"method": method, "engine": engine, **params} self.cluster_manager = ClusterComputationManager(tracks, feats, self._log) - for name in ("Interactive – KMeans", "Interactive – HDBSCAN"): - if name in self.plugin_views: - try: - self.plugin_views[name].destroy() - except Exception: - pass - self.plugin_views.pop(name, None) - def done(): self.cluster_generation_running = False messagebox.showinfo("Clustered Playlists", "Generation complete") From 7708577391ceda8e81ba20a47148b168dc1505ec Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 15 Dec 2025 22:49:46 -0500 Subject: [PATCH 077/606] Trigger post-cluster resize for graph canvas --- cluster_graph_panel.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/cluster_graph_panel.py b/cluster_graph_panel.py index 20a5ff5..091d5d8 100644 --- a/cluster_graph_panel.py +++ b/cluster_graph_panel.py @@ -407,6 +407,7 @@ def _on_clusters_ready(self, labels, params: dict): if self._loading_lbl.winfo_exists(): self._loading_lbl.pack_forget() self._render_clusters(labels) + self._force_post_cluster_resize() log_cluster_summary(labels, self.log) if duration_ms is not None: logger.info( @@ -490,6 +491,26 @@ def final_redraw(): self.after_idle(final_redraw) self.after(30, final_redraw) + def _force_post_cluster_resize(self) -> None: + """Trigger a resize after clustering to avoid oversized initial renders.""" + + widget = self.canvas.get_tk_widget() + + def _resize_once() -> None: + try: + widget.update_idletasks() + except Exception: + pass + + w = widget.winfo_width() + h = widget.winfo_height() + if w <= 1 or h <= 1: + return + self.on_resize(w, h) + + self.after_idle(_resize_once) + self.after(30, _resize_once) + def on_resize(self, width: int, height: int) -> None: """Resize the matplotlib canvas to the given dimensions.""" From 6dcf6e8c4658bcb4e98b26d5613585cd17ff84d9 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 15 Dec 2025 22:54:24 -0500 Subject: [PATCH 078/606] Auto-select interactive clustering panels --- main_gui.py | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/main_gui.py b/main_gui.py index 64c750a..2a7fd3b 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1419,6 +1419,7 @@ def __init__(self): self.cluster_data = None self.cluster_manager = None self.cluster_generation_running = False + self.cluster_params = None self.folder_filter = {"include": [], "exclude": []} # Shared audio feature/analysis engine selection @@ -1711,6 +1712,9 @@ def build_ui(self): self.plugin_list.insert("end", name) self.plugin_list.bind("<>", self.on_plugin_select) + self._select_default_plugin() + self.on_plugin_select(None) + self.plugin_panel = ttk.Frame(self.playlist_tab) self.plugin_panel.grid(row=0, column=1, sticky="nsew", padx=10, pady=10) self.playlist_tab.columnconfigure(1, weight=1) @@ -2111,7 +2115,9 @@ def on_plugin_select(self, event): try: sel = self.plugin_list.get(self.plugin_list.curselection()) except tk.TclError: - return + sel = self._select_default_plugin() + if not sel: + return logging.info("[perf] tool switch -> %s", sel) if self.active_plugin and self.active_plugin in self.plugin_views: @@ -2795,6 +2801,35 @@ def _refresh_plugin_panel(self): self.active_plugin = None self.on_plugin_select(None) + def _select_default_plugin(self) -> str | None: + """Ensure a playlist plugin is selected and return its name.""" + if not hasattr(self, "plugin_list"): + return None + + preferred = None + method = (self.cluster_params or {}).get("method") if hasattr(self, "cluster_params") else None + if method == "hdbscan": + preferred = "Interactive – HDBSCAN" + elif method == "kmeans": + preferred = "Interactive – KMeans" + + names = list(self.plugin_list.get(0, "end")) + if not preferred and names: + preferred = names[0] + + if preferred is None: + return None + + try: + self.plugin_list.selection_clear(0, "end") + index = names.index(preferred) + self.plugin_list.selection_set(index) + self.plugin_list.activate(index) + self.plugin_list.see(index) + return preferred + except (tk.TclError, ValueError): + return None + def _tagfix_filter_dialog(self): dlg = tk.Toplevel(self) dlg.title("Exclude from this scan") From 939ae0bb79f3d27a5258b64f94f7de6e8b9c63f9 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 15 Dec 2025 22:54:56 -0500 Subject: [PATCH 079/606] Revert "Auto-select interactive clustering panels" --- main_gui.py | 37 +------------------------------------ 1 file changed, 1 insertion(+), 36 deletions(-) diff --git a/main_gui.py b/main_gui.py index 2a7fd3b..64c750a 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1419,7 +1419,6 @@ def __init__(self): self.cluster_data = None self.cluster_manager = None self.cluster_generation_running = False - self.cluster_params = None self.folder_filter = {"include": [], "exclude": []} # Shared audio feature/analysis engine selection @@ -1712,9 +1711,6 @@ def build_ui(self): self.plugin_list.insert("end", name) self.plugin_list.bind("<>", self.on_plugin_select) - self._select_default_plugin() - self.on_plugin_select(None) - self.plugin_panel = ttk.Frame(self.playlist_tab) self.plugin_panel.grid(row=0, column=1, sticky="nsew", padx=10, pady=10) self.playlist_tab.columnconfigure(1, weight=1) @@ -2115,9 +2111,7 @@ def on_plugin_select(self, event): try: sel = self.plugin_list.get(self.plugin_list.curselection()) except tk.TclError: - sel = self._select_default_plugin() - if not sel: - return + return logging.info("[perf] tool switch -> %s", sel) if self.active_plugin and self.active_plugin in self.plugin_views: @@ -2801,35 +2795,6 @@ def _refresh_plugin_panel(self): self.active_plugin = None self.on_plugin_select(None) - def _select_default_plugin(self) -> str | None: - """Ensure a playlist plugin is selected and return its name.""" - if not hasattr(self, "plugin_list"): - return None - - preferred = None - method = (self.cluster_params or {}).get("method") if hasattr(self, "cluster_params") else None - if method == "hdbscan": - preferred = "Interactive – HDBSCAN" - elif method == "kmeans": - preferred = "Interactive – KMeans" - - names = list(self.plugin_list.get(0, "end")) - if not preferred and names: - preferred = names[0] - - if preferred is None: - return None - - try: - self.plugin_list.selection_clear(0, "end") - index = names.index(preferred) - self.plugin_list.selection_set(index) - self.plugin_list.activate(index) - self.plugin_list.see(index) - return preferred - except (tk.TclError, ValueError): - return None - def _tagfix_filter_dialog(self): dlg = tk.Toplevel(self) dlg.title("Exclude from this scan") From e15dde363bf1fd6c89bb33044707c18a43d09493 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 16 Dec 2025 09:45:09 -0500 Subject: [PATCH 080/606] Improve lasso flow and add clustering guidance --- cluster_graph_panel.py | 28 ++++++++++++++++++++++++++++ main_gui.py | 28 +++++++++++++++++++++++++++- 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/cluster_graph_panel.py b/cluster_graph_panel.py index 091d5d8..81fd3f6 100644 --- a/cluster_graph_panel.py +++ b/cluster_graph_panel.py @@ -81,6 +81,7 @@ def __init__( self._remove_lasso_started = False self._remove_lasso_prev_state = False self._remove_lasso_confirm_pending = False + self._add_highlight_pending = False # Hover support widgets will be set later via ``setup_hover`` self.hover_panel = None @@ -151,6 +152,8 @@ def _set_lasso_enabled(self, active: bool): """Toggle matplotlib lasso state and sync UI controls.""" self._clear_selection() + if not active: + self._add_highlight_pending = False if active: if self.lasso is None: self.lasso = LassoSelector(self.ax, onselect=self._on_lasso_select) @@ -253,6 +256,12 @@ def _on_lasso_select(self, verts): if not self.selected_indices: self.ok_btn.configure(state="disabled") return + if self._add_highlight_pending and self.selected_indices: + self.selected_tracks = [self.tracks[i] for i in self.selected_indices] + self.add_highlight_to_temp() + self._add_highlight_pending = False + self._set_lasso_enabled(False) + return if self.selected_indices: self.ok_btn.configure(state="normal") else: @@ -678,6 +687,25 @@ def remove_selected_from_temp(self): self._begin_temp_lasso_removal() + def begin_add_highlight_flow(self) -> None: + """Activate lasso (if needed) before adding highlights to the temp list.""" + + if self._remove_lasso_started: + self._cancel_temp_lasso_removal() + + if not self.selected_indices: + self._add_highlight_pending = True + self._set_lasso_enabled(True) + self.log( + "Lasso enabled – draw around songs to add to the temporary playlist" + ) + return + + self.selected_tracks = [self.tracks[i] for i in self.selected_indices] + self.add_highlight_to_temp() + if self.lasso is not None: + self._set_lasso_enabled(False) + def _begin_temp_lasso_removal(self) -> None: """Switch to lasso mode to pick songs for removal.""" diff --git a/main_gui.py b/main_gui.py index 64c750a..600b848 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1051,7 +1051,7 @@ def _load_cluster(): add_highlight_btn = ttk.Button( temp_box, text="Add Highlighted Songs", - command=lambda: panel and panel.add_highlight_to_temp(), + command=lambda: panel and panel.begin_add_highlight_flow(), ) add_highlight_btn.grid(row=2, column=0, sticky="ew", padx=5, pady=(0, 5)) @@ -1127,6 +1127,32 @@ def _auto_create_all(): ) redo_btn.pack(side="left", padx=(5, 0)) + guidance_messages = { + "Interactive – KMeans": ( + "Guide:\n" + "• Run Clusters to generate or refresh KMeans groups.\n" + "• Load Cluster or click Add Highlighted Songs to auto-enable the lasso and pick songs for the temp playlist.\n" + "• Show All re-highlights the temp playlist; Remove Selected uses lasso to prune; Create Playlist saves it." + ), + "Interactive – HDBSCAN": ( + "Guide:\n" + "• Run Clusters (or Redo Values) to explore HDBSCAN groupings.\n" + "• Load Cluster or use Add Highlighted Songs to turn on lasso mode and add a custom selection to the temp playlist.\n" + "• Show All re-highlights the temp playlist; Remove Selected lasso-removes songs; Create Playlist exports it." + ), + } + + guidance_text = guidance_messages.get(name) + if guidance_text: + guide_lbl = ttk.Label( + side_tools, + text=guidance_text, + justify="left", + wraplength=250, + foreground="gray", + ) + guide_lbl.grid(row=4, column=0, sticky="ew", padx=5, pady=(5, 0)) + placeholder: ttk.Label | None = None def _sync_cluster_controls(cluster_ready: bool, running: bool) -> None: From 21c985f071b6ca2c0b08194aca6457332d91373c Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 16 Dec 2025 09:54:13 -0500 Subject: [PATCH 081/606] Add live message board for cluster controls --- main_gui.py | 91 ++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 72 insertions(+), 19 deletions(-) diff --git a/main_gui.py b/main_gui.py index 600b848..3e7859b 100644 --- a/main_gui.py +++ b/main_gui.py @@ -50,7 +50,7 @@ from PIL import Image, ImageTk from mutagen import File as MutagenFile from tag_fixer import MIN_INTERACTIVE_SCORE, FileRecord -from typing import Callable, List +from typing import Any, Callable, List from indexer_control import cancel_event, IndexCancelled import library_sync import playlist_generator @@ -939,6 +939,23 @@ def generate(): panel: ClusterGraphPanel | None = None + message_var = tk.StringVar(value="") + + def _set_message(text: str) -> None: + message_var.set(text) + + def _clear_message(event: tk.Event | None = None) -> None: # type: ignore[name-defined] + message_var.set("") + + def _message_action(text: str, action: Callable[[], Any]) -> Callable[[], Any]: + def wrapped() -> Any: + _set_message(text) + return action() + + return wrapped + + frame.bind_all("", _clear_message) + side_tools = ttk.Frame(container) side_tools.grid(row=0, column=1, rowspan=3, sticky="ns", padx=(10, 0)) side_tools.columnconfigure(0, weight=1) @@ -955,7 +972,10 @@ def generate(): run_btn = ttk.Button( control_banner, text="Run Clusters", - command=lambda: app.cluster_playlists_dialog(params["method"]), + command=_message_action( + "Running clustering will refresh your groups based on the current settings.", + lambda: app.cluster_playlists_dialog(params["method"]), + ), ) run_btn.grid(row=0, column=0, padx=5, pady=5, sticky="ew") @@ -965,7 +985,10 @@ def generate(): playlist_btn = ttk.Button( side_tools, text="Current Playlists", - command=lambda: panel and panel.show_current_playlists(), + command=_message_action( + "Listing the playlists you already have for quick review.", + lambda: panel and panel.show_current_playlists(), + ), ) playlist_btn.grid(row=1, column=0, sticky="ew", pady=(0, 10)) @@ -996,6 +1019,7 @@ def generate(): manual_entry.grid(row=1, column=1, sticky="ew", padx=5, pady=(0, 2)) def _load_cluster(): + _set_message("Loading the selected cluster and preparing lasso controls.") if panel is None: return choice = manual_cluster_var.get().strip() or cluster_select_var.get().strip() @@ -1044,28 +1068,40 @@ def _load_cluster(): show_all_btn = ttk.Button( temp_box, text="Show All", - command=lambda: panel and panel.highlight_temp_playlist(), + command=_message_action( + "Highlighting all songs currently in the temporary playlist.", + lambda: panel and panel.highlight_temp_playlist(), + ), ) show_all_btn.grid(row=1, column=0, sticky="ew", padx=5, pady=(0, 5)) add_highlight_btn = ttk.Button( temp_box, text="Add Highlighted Songs", - command=lambda: panel and panel.begin_add_highlight_flow(), + command=_message_action( + "Enable lasso mode to add the highlighted graph selection to the temp playlist.", + lambda: panel and panel.begin_add_highlight_flow(), + ), ) add_highlight_btn.grid(row=2, column=0, sticky="ew", padx=5, pady=(0, 5)) temp_remove_btn = ttk.Button( temp_box, text="Remove Selected", - command=lambda: panel and panel.remove_selected_from_temp(), + command=_message_action( + "Removing the selected songs from the temporary playlist.", + lambda: panel and panel.remove_selected_from_temp(), + ), ) temp_remove_btn.grid(row=3, column=0, sticky="ew", padx=5, pady=(0, 5)) create_playlist_btn = ttk.Button( temp_box, text="Create Playlist", - command=lambda: panel and panel.create_temp_playlist(), + command=_message_action( + "Saving the current temporary playlist as a new playlist.", + lambda: panel and panel.create_temp_playlist(), + ), ) create_playlist_btn.grid(row=4, column=0, sticky="ew", padx=5, pady=(0, 5)) @@ -1080,25 +1116,35 @@ def _load_cluster(): btn_frame, text="Lasso Mode", variable=lasso_var, - command=lambda: panel and panel.toggle_lasso(), + command=_message_action( + "Toggling lasso mode for manual graph selection.", + lambda: panel and panel.toggle_lasso(), + ), ) lasso_btn.pack(side="left") ok_btn = ttk.Button( btn_frame, text="OK", - command=lambda: panel and panel.finalize_lasso(), + command=_message_action( + "Finalizing the current lasso selection into the temp playlist.", + lambda: panel and panel.finalize_lasso(), + ), ) ok_btn.pack(side="left", padx=(5, 0)) gen_btn = ttk.Button( btn_frame, text="Generate Playlist", - command=lambda: panel and panel.create_playlist(), + command=_message_action( + "Building a playlist from the current selection.", + lambda: panel and panel.create_playlist(), + ), ) gen_btn.pack(side="left", padx=(5, 0)) def _auto_create_all(): + _set_message("Auto-creating playlists for every available cluster.") if panel is None or not getattr(panel, "cluster_params", None): return method = panel.cluster_params.get("method") @@ -1123,7 +1169,12 @@ def _auto_create_all(): redo_btn: ttk.Button | None = None if name == "Interactive – HDBSCAN": redo_btn = ttk.Button( - btn_frame, text="Redo Values", command=lambda: panel and panel.open_param_dialog() + btn_frame, + text="Redo Values", + command=_message_action( + "Reopen the clustering parameters to tweak HDBSCAN values.", + lambda: panel and panel.open_param_dialog(), + ), ) redo_btn.pack(side="left", padx=(5, 0)) @@ -1144,14 +1195,16 @@ def _auto_create_all(): guidance_text = guidance_messages.get(name) if guidance_text: - guide_lbl = ttk.Label( - side_tools, - text=guidance_text, - justify="left", - wraplength=250, - foreground="gray", - ) - guide_lbl.grid(row=4, column=0, sticky="ew", padx=5, pady=(5, 0)) + _set_message(guidance_text) + + guide_lbl = ttk.Label( + side_tools, + textvariable=message_var, + justify="left", + wraplength=250, + foreground="gray", + ) + guide_lbl.grid(row=4, column=0, sticky="ew", padx=5, pady=(5, 0)) placeholder: ttk.Label | None = None From c7553ba6a3d0b2c490e0b18ca85b3570aacc9e70 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 16 Dec 2025 10:02:14 -0500 Subject: [PATCH 082/606] Relocate graph messaging to banner --- main_gui.py | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/main_gui.py b/main_gui.py index 3e7859b..190e920 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1166,18 +1166,6 @@ def _auto_create_all(): auto_btn = ttk.Button(btn_frame, text="Auto-Create", command=_auto_create_all) auto_btn.pack(side="left", padx=(5, 0)) - redo_btn: ttk.Button | None = None - if name == "Interactive – HDBSCAN": - redo_btn = ttk.Button( - btn_frame, - text="Redo Values", - command=_message_action( - "Reopen the clustering parameters to tweak HDBSCAN values.", - lambda: panel and panel.open_param_dialog(), - ), - ) - redo_btn.pack(side="left", padx=(5, 0)) - guidance_messages = { "Interactive – KMeans": ( "Guide:\n" @@ -1187,7 +1175,7 @@ def _auto_create_all(): ), "Interactive – HDBSCAN": ( "Guide:\n" - "• Run Clusters (or Redo Values) to explore HDBSCAN groupings.\n" + "• Run Clusters to explore HDBSCAN groupings.\n" "• Load Cluster or use Add Highlighted Songs to turn on lasso mode and add a custom selection to the temp playlist.\n" "• Show All re-highlights the temp playlist; Remove Selected lasso-removes songs; Create Playlist exports it." ), @@ -1197,14 +1185,15 @@ def _auto_create_all(): if guidance_text: _set_message(guidance_text) - guide_lbl = ttk.Label( - side_tools, + message_lbl = ttk.Label( + btn_frame, textvariable=message_var, justify="left", - wraplength=250, + wraplength=450, foreground="gray", + anchor="w", ) - guide_lbl.grid(row=4, column=0, sticky="ew", padx=5, pady=(5, 0)) + message_lbl.pack(side="left", padx=(10, 0), fill="x", expand=True) placeholder: ttk.Label | None = None @@ -1229,9 +1218,6 @@ def _sync_cluster_controls(cluster_ready: bool, running: bool) -> None: ): widget.config(state=state) - if redo_btn is not None: - redo_btn.config(state=state) - run_btn.state(["disabled"] if running else ["!disabled"]) if running: From 0e9544f5b812554c38a542b8bf152723e908b55a Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 16 Dec 2025 10:03:05 -0500 Subject: [PATCH 083/606] Revert "Move graph status messaging into control banner" --- main_gui.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/main_gui.py b/main_gui.py index 190e920..3e7859b 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1166,6 +1166,18 @@ def _auto_create_all(): auto_btn = ttk.Button(btn_frame, text="Auto-Create", command=_auto_create_all) auto_btn.pack(side="left", padx=(5, 0)) + redo_btn: ttk.Button | None = None + if name == "Interactive – HDBSCAN": + redo_btn = ttk.Button( + btn_frame, + text="Redo Values", + command=_message_action( + "Reopen the clustering parameters to tweak HDBSCAN values.", + lambda: panel and panel.open_param_dialog(), + ), + ) + redo_btn.pack(side="left", padx=(5, 0)) + guidance_messages = { "Interactive – KMeans": ( "Guide:\n" @@ -1175,7 +1187,7 @@ def _auto_create_all(): ), "Interactive – HDBSCAN": ( "Guide:\n" - "• Run Clusters to explore HDBSCAN groupings.\n" + "• Run Clusters (or Redo Values) to explore HDBSCAN groupings.\n" "• Load Cluster or use Add Highlighted Songs to turn on lasso mode and add a custom selection to the temp playlist.\n" "• Show All re-highlights the temp playlist; Remove Selected lasso-removes songs; Create Playlist exports it." ), @@ -1185,15 +1197,14 @@ def _auto_create_all(): if guidance_text: _set_message(guidance_text) - message_lbl = ttk.Label( - btn_frame, + guide_lbl = ttk.Label( + side_tools, textvariable=message_var, justify="left", - wraplength=450, + wraplength=250, foreground="gray", - anchor="w", ) - message_lbl.pack(side="left", padx=(10, 0), fill="x", expand=True) + guide_lbl.grid(row=4, column=0, sticky="ew", padx=5, pady=(5, 0)) placeholder: ttk.Label | None = None @@ -1218,6 +1229,9 @@ def _sync_cluster_controls(cluster_ready: bool, running: bool) -> None: ): widget.config(state=state) + if redo_btn is not None: + redo_btn.config(state=state) + run_btn.state(["disabled"] if running else ["!disabled"]) if running: From e980f9d877ba4b8bbc62515206a093aebda9596b Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 16 Dec 2025 10:26:14 -0500 Subject: [PATCH 084/606] Refine playlist creator banner layout --- main_gui.py | 99 +++++++++++++++++++++++++++++++++-------------------- 1 file changed, 61 insertions(+), 38 deletions(-) diff --git a/main_gui.py b/main_gui.py index 3e7859b..3c7ae66 100644 --- a/main_gui.py +++ b/main_gui.py @@ -18,6 +18,7 @@ pass import tkinter as tk +import tkinter.font as tkfont from tkinter import ttk Style = ttk.Style # Default built-in themes @@ -940,12 +941,53 @@ def generate(): panel: ClusterGraphPanel | None = None message_var = tk.StringVar(value="") + action_message = "" + status_message = "" + message_label: ttk.Label | None = None + + def _normalize_message(text: str) -> str: + return " ".join(text.split()) + + def _refresh_banner_message(event: tk.Event | None = None) -> None: # type: ignore[name-defined] + active_text = action_message + if status_message: + active_text = f"{active_text} — {status_message}" if active_text else status_message + if not active_text: + message_var.set("") + return + + if message_label is None or not message_label.winfo_exists(): + message_var.set(active_text) + return + + label_width = message_label.winfo_width() + if label_width <= 1: + message_var.set(active_text) + return + + font = tkfont.nametofont(message_label.cget("font")) + if font.measure(active_text) <= label_width: + message_var.set(active_text) + return + + truncated = active_text + ellipsis = "…" + while truncated and font.measure(truncated + ellipsis) > label_width: + truncated = truncated[:-1] + message_var.set(truncated + ellipsis if truncated else ellipsis) def _set_message(text: str) -> None: - message_var.set(text) + nonlocal action_message + action_message = _normalize_message(text) + _refresh_banner_message() + + def _set_status(text: str) -> None: + nonlocal status_message + status_message = _normalize_message(text) + _refresh_banner_message() def _clear_message(event: tk.Event | None = None) -> None: # type: ignore[name-defined] - message_var.set("") + _set_message("") def _message_action(text: str, action: Callable[[], Any]) -> Callable[[], Any]: def wrapped() -> Any: @@ -967,7 +1009,6 @@ def wrapped() -> Any: control_banner = ttk.Frame(side_tools) control_banner.grid(row=0, column=0, sticky="ew", pady=(0, 10)) - control_banner.columnconfigure(1, weight=1) run_btn = ttk.Button( control_banner, @@ -979,9 +1020,6 @@ def wrapped() -> Any: ) run_btn.grid(row=0, column=0, padx=5, pady=5, sticky="ew") - status = ttk.Label(control_banner) - status.grid(row=0, column=1, padx=5, pady=5, sticky="w") - playlist_btn = ttk.Button( side_tools, text="Current Playlists", @@ -1107,8 +1145,12 @@ def _load_cluster(): ttk.Separator(container, orient="horizontal").grid(row=1, column=0, sticky="ew") - btn_frame = ttk.Frame(container) + btn_frame = ttk.Frame(container, height=40) btn_frame.grid(row=2, column=0, sticky="ew", pady=5) + btn_frame.grid_propagate(False) + for col in range(4): + btn_frame.columnconfigure(col, weight=0) + btn_frame.columnconfigure(4, weight=1) lasso_var = tk.BooleanVar(value=False) @@ -1121,7 +1163,7 @@ def _load_cluster(): lambda: panel and panel.toggle_lasso(), ), ) - lasso_btn.pack(side="left") + lasso_btn.grid(row=0, column=0, padx=(0, 5), pady=5, sticky="w") ok_btn = ttk.Button( btn_frame, @@ -1131,7 +1173,7 @@ def _load_cluster(): lambda: panel and panel.finalize_lasso(), ), ) - ok_btn.pack(side="left", padx=(5, 0)) + ok_btn.grid(row=0, column=1, padx=(0, 5), pady=5, sticky="w") gen_btn = ttk.Button( btn_frame, @@ -1141,7 +1183,7 @@ def _load_cluster(): lambda: panel and panel.create_playlist(), ), ) - gen_btn.pack(side="left", padx=(5, 0)) + gen_btn.grid(row=0, column=2, padx=(0, 5), pady=5, sticky="w") def _auto_create_all(): _set_message("Auto-creating playlists for every available cluster.") @@ -1164,19 +1206,12 @@ def _auto_create_all(): ).start() auto_btn = ttk.Button(btn_frame, text="Auto-Create", command=_auto_create_all) - auto_btn.pack(side="left", padx=(5, 0)) + auto_btn.grid(row=0, column=3, padx=(0, 10), pady=5, sticky="w") - redo_btn: ttk.Button | None = None - if name == "Interactive – HDBSCAN": - redo_btn = ttk.Button( - btn_frame, - text="Redo Values", - command=_message_action( - "Reopen the clustering parameters to tweak HDBSCAN values.", - lambda: panel and panel.open_param_dialog(), - ), - ) - redo_btn.pack(side="left", padx=(5, 0)) + message_label = ttk.Label(btn_frame, textvariable=message_var, anchor="w") + message_label.grid(row=0, column=4, padx=(0, 5), pady=5, sticky="ew") + message_label.bind("", _refresh_banner_message) + _refresh_banner_message() guidance_messages = { "Interactive – KMeans": ( @@ -1187,7 +1222,7 @@ def _auto_create_all(): ), "Interactive – HDBSCAN": ( "Guide:\n" - "• Run Clusters (or Redo Values) to explore HDBSCAN groupings.\n" + "• Run Clusters to explore HDBSCAN groupings.\n" "• Load Cluster or use Add Highlighted Songs to turn on lasso mode and add a custom selection to the temp playlist.\n" "• Show All re-highlights the temp playlist; Remove Selected lasso-removes songs; Create Playlist exports it." ), @@ -1197,15 +1232,6 @@ def _auto_create_all(): if guidance_text: _set_message(guidance_text) - guide_lbl = ttk.Label( - side_tools, - textvariable=message_var, - justify="left", - wraplength=250, - foreground="gray", - ) - guide_lbl.grid(row=4, column=0, sticky="ew", padx=5, pady=(5, 0)) - placeholder: ttk.Label | None = None def _sync_cluster_controls(cluster_ready: bool, running: bool) -> None: @@ -1229,17 +1255,14 @@ def _sync_cluster_controls(cluster_ready: bool, running: bool) -> None: ): widget.config(state=state) - if redo_btn is not None: - redo_btn.config(state=state) - run_btn.state(["disabled"] if running else ["!disabled"]) if running: - status.config(text="Clustering in progress…") + _set_status("Clustering in progress…") elif not ready: - status.config(text="Run clustering once first") + _set_status("Run clustering once first") else: - status.config(text="Clusters loaded") + _set_status("Clusters loaded") def _ensure_placeholder(message: str) -> ttk.Label: nonlocal placeholder From 4dd3504af2a369fb67f131282bbe7943ddfbfc3e Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 16 Dec 2025 10:27:18 -0500 Subject: [PATCH 085/606] Revert "Refine playlist creator banner layout" --- main_gui.py | 99 ++++++++++++++++++++--------------------------------- 1 file changed, 38 insertions(+), 61 deletions(-) diff --git a/main_gui.py b/main_gui.py index 3c7ae66..3e7859b 100644 --- a/main_gui.py +++ b/main_gui.py @@ -18,7 +18,6 @@ pass import tkinter as tk -import tkinter.font as tkfont from tkinter import ttk Style = ttk.Style # Default built-in themes @@ -941,53 +940,12 @@ def generate(): panel: ClusterGraphPanel | None = None message_var = tk.StringVar(value="") - action_message = "" - status_message = "" - message_label: ttk.Label | None = None - - def _normalize_message(text: str) -> str: - return " ".join(text.split()) - - def _refresh_banner_message(event: tk.Event | None = None) -> None: # type: ignore[name-defined] - active_text = action_message - if status_message: - active_text = f"{active_text} — {status_message}" if active_text else status_message - if not active_text: - message_var.set("") - return - - if message_label is None or not message_label.winfo_exists(): - message_var.set(active_text) - return - - label_width = message_label.winfo_width() - if label_width <= 1: - message_var.set(active_text) - return - - font = tkfont.nametofont(message_label.cget("font")) - if font.measure(active_text) <= label_width: - message_var.set(active_text) - return - - truncated = active_text - ellipsis = "…" - while truncated and font.measure(truncated + ellipsis) > label_width: - truncated = truncated[:-1] - message_var.set(truncated + ellipsis if truncated else ellipsis) def _set_message(text: str) -> None: - nonlocal action_message - action_message = _normalize_message(text) - _refresh_banner_message() - - def _set_status(text: str) -> None: - nonlocal status_message - status_message = _normalize_message(text) - _refresh_banner_message() + message_var.set(text) def _clear_message(event: tk.Event | None = None) -> None: # type: ignore[name-defined] - _set_message("") + message_var.set("") def _message_action(text: str, action: Callable[[], Any]) -> Callable[[], Any]: def wrapped() -> Any: @@ -1009,6 +967,7 @@ def wrapped() -> Any: control_banner = ttk.Frame(side_tools) control_banner.grid(row=0, column=0, sticky="ew", pady=(0, 10)) + control_banner.columnconfigure(1, weight=1) run_btn = ttk.Button( control_banner, @@ -1020,6 +979,9 @@ def wrapped() -> Any: ) run_btn.grid(row=0, column=0, padx=5, pady=5, sticky="ew") + status = ttk.Label(control_banner) + status.grid(row=0, column=1, padx=5, pady=5, sticky="w") + playlist_btn = ttk.Button( side_tools, text="Current Playlists", @@ -1145,12 +1107,8 @@ def _load_cluster(): ttk.Separator(container, orient="horizontal").grid(row=1, column=0, sticky="ew") - btn_frame = ttk.Frame(container, height=40) + btn_frame = ttk.Frame(container) btn_frame.grid(row=2, column=0, sticky="ew", pady=5) - btn_frame.grid_propagate(False) - for col in range(4): - btn_frame.columnconfigure(col, weight=0) - btn_frame.columnconfigure(4, weight=1) lasso_var = tk.BooleanVar(value=False) @@ -1163,7 +1121,7 @@ def _load_cluster(): lambda: panel and panel.toggle_lasso(), ), ) - lasso_btn.grid(row=0, column=0, padx=(0, 5), pady=5, sticky="w") + lasso_btn.pack(side="left") ok_btn = ttk.Button( btn_frame, @@ -1173,7 +1131,7 @@ def _load_cluster(): lambda: panel and panel.finalize_lasso(), ), ) - ok_btn.grid(row=0, column=1, padx=(0, 5), pady=5, sticky="w") + ok_btn.pack(side="left", padx=(5, 0)) gen_btn = ttk.Button( btn_frame, @@ -1183,7 +1141,7 @@ def _load_cluster(): lambda: panel and panel.create_playlist(), ), ) - gen_btn.grid(row=0, column=2, padx=(0, 5), pady=5, sticky="w") + gen_btn.pack(side="left", padx=(5, 0)) def _auto_create_all(): _set_message("Auto-creating playlists for every available cluster.") @@ -1206,12 +1164,19 @@ def _auto_create_all(): ).start() auto_btn = ttk.Button(btn_frame, text="Auto-Create", command=_auto_create_all) - auto_btn.grid(row=0, column=3, padx=(0, 10), pady=5, sticky="w") + auto_btn.pack(side="left", padx=(5, 0)) - message_label = ttk.Label(btn_frame, textvariable=message_var, anchor="w") - message_label.grid(row=0, column=4, padx=(0, 5), pady=5, sticky="ew") - message_label.bind("", _refresh_banner_message) - _refresh_banner_message() + redo_btn: ttk.Button | None = None + if name == "Interactive – HDBSCAN": + redo_btn = ttk.Button( + btn_frame, + text="Redo Values", + command=_message_action( + "Reopen the clustering parameters to tweak HDBSCAN values.", + lambda: panel and panel.open_param_dialog(), + ), + ) + redo_btn.pack(side="left", padx=(5, 0)) guidance_messages = { "Interactive – KMeans": ( @@ -1222,7 +1187,7 @@ def _auto_create_all(): ), "Interactive – HDBSCAN": ( "Guide:\n" - "• Run Clusters to explore HDBSCAN groupings.\n" + "• Run Clusters (or Redo Values) to explore HDBSCAN groupings.\n" "• Load Cluster or use Add Highlighted Songs to turn on lasso mode and add a custom selection to the temp playlist.\n" "• Show All re-highlights the temp playlist; Remove Selected lasso-removes songs; Create Playlist exports it." ), @@ -1232,6 +1197,15 @@ def _auto_create_all(): if guidance_text: _set_message(guidance_text) + guide_lbl = ttk.Label( + side_tools, + textvariable=message_var, + justify="left", + wraplength=250, + foreground="gray", + ) + guide_lbl.grid(row=4, column=0, sticky="ew", padx=5, pady=(5, 0)) + placeholder: ttk.Label | None = None def _sync_cluster_controls(cluster_ready: bool, running: bool) -> None: @@ -1255,14 +1229,17 @@ def _sync_cluster_controls(cluster_ready: bool, running: bool) -> None: ): widget.config(state=state) + if redo_btn is not None: + redo_btn.config(state=state) + run_btn.state(["disabled"] if running else ["!disabled"]) if running: - _set_status("Clustering in progress…") + status.config(text="Clustering in progress…") elif not ready: - _set_status("Run clustering once first") + status.config(text="Run clustering once first") else: - _set_status("Clusters loaded") + status.config(text="Clusters loaded") def _ensure_placeholder(message: str) -> ttk.Label: nonlocal placeholder From 09baeffddb51e29265a08b1e8fbdaa167cddbc55 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 16 Dec 2025 10:35:04 -0500 Subject: [PATCH 086/606] Remove clustering guide message --- main_gui.py | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/main_gui.py b/main_gui.py index 3e7859b..a72c6f5 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1178,34 +1178,6 @@ def _auto_create_all(): ) redo_btn.pack(side="left", padx=(5, 0)) - guidance_messages = { - "Interactive – KMeans": ( - "Guide:\n" - "• Run Clusters to generate or refresh KMeans groups.\n" - "• Load Cluster or click Add Highlighted Songs to auto-enable the lasso and pick songs for the temp playlist.\n" - "• Show All re-highlights the temp playlist; Remove Selected uses lasso to prune; Create Playlist saves it." - ), - "Interactive – HDBSCAN": ( - "Guide:\n" - "• Run Clusters (or Redo Values) to explore HDBSCAN groupings.\n" - "• Load Cluster or use Add Highlighted Songs to turn on lasso mode and add a custom selection to the temp playlist.\n" - "• Show All re-highlights the temp playlist; Remove Selected lasso-removes songs; Create Playlist exports it." - ), - } - - guidance_text = guidance_messages.get(name) - if guidance_text: - _set_message(guidance_text) - - guide_lbl = ttk.Label( - side_tools, - textvariable=message_var, - justify="left", - wraplength=250, - foreground="gray", - ) - guide_lbl.grid(row=4, column=0, sticky="ew", padx=5, pady=(5, 0)) - placeholder: ttk.Label | None = None def _sync_cluster_controls(cluster_ready: bool, running: bool) -> None: From 7260e7d9ea936cac364d80b10d9d59356f540e12 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 16 Dec 2025 10:42:25 -0500 Subject: [PATCH 087/606] Add right panel action counter --- main_gui.py | 64 +++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 45 insertions(+), 19 deletions(-) diff --git a/main_gui.py b/main_gui.py index a72c6f5..6e843ad 100644 --- a/main_gui.py +++ b/main_gui.py @@ -940,6 +940,8 @@ def generate(): panel: ClusterGraphPanel | None = None message_var = tk.StringVar(value="") + banner_var = tk.StringVar(value="") + press_counter = 0 def _set_message(text: str) -> None: message_var.set(text) @@ -954,6 +956,16 @@ def wrapped() -> Any: return wrapped + def _right_panel_action(action: Callable[[], Any] | None) -> Callable[[], Any]: + def wrapped() -> Any: + nonlocal press_counter + press_counter += 1 + banner_var.set(str(press_counter)) + if action is not None: + return action() + + return wrapped + frame.bind_all("", _clear_message) side_tools = ttk.Frame(container) @@ -972,9 +984,11 @@ def wrapped() -> Any: run_btn = ttk.Button( control_banner, text="Run Clusters", - command=_message_action( - "Running clustering will refresh your groups based on the current settings.", - lambda: app.cluster_playlists_dialog(params["method"]), + command=_right_panel_action( + _message_action( + "Running clustering will refresh your groups based on the current settings.", + lambda: app.cluster_playlists_dialog(params["method"]), + ) ), ) run_btn.grid(row=0, column=0, padx=5, pady=5, sticky="ew") @@ -985,9 +999,11 @@ def wrapped() -> Any: playlist_btn = ttk.Button( side_tools, text="Current Playlists", - command=_message_action( - "Listing the playlists you already have for quick review.", - lambda: panel and panel.show_current_playlists(), + command=_right_panel_action( + _message_action( + "Listing the playlists you already have for quick review.", + lambda: panel and panel.show_current_playlists(), + ) ), ) playlist_btn.grid(row=1, column=0, sticky="ew", pady=(0, 10)) @@ -1036,7 +1052,7 @@ def _load_cluster(): load_btn = ttk.Button( cluster_box, text="Load Cluster", - command=_load_cluster, + command=_right_panel_action(_load_cluster), ) load_btn.grid(row=2, column=0, columnspan=2, sticky="ew", padx=5, pady=(0, 5)) @@ -1068,9 +1084,11 @@ def _load_cluster(): show_all_btn = ttk.Button( temp_box, text="Show All", - command=_message_action( - "Highlighting all songs currently in the temporary playlist.", - lambda: panel and panel.highlight_temp_playlist(), + command=_right_panel_action( + _message_action( + "Highlighting all songs currently in the temporary playlist.", + lambda: panel and panel.highlight_temp_playlist(), + ) ), ) show_all_btn.grid(row=1, column=0, sticky="ew", padx=5, pady=(0, 5)) @@ -1078,9 +1096,11 @@ def _load_cluster(): add_highlight_btn = ttk.Button( temp_box, text="Add Highlighted Songs", - command=_message_action( - "Enable lasso mode to add the highlighted graph selection to the temp playlist.", - lambda: panel and panel.begin_add_highlight_flow(), + command=_right_panel_action( + _message_action( + "Enable lasso mode to add the highlighted graph selection to the temp playlist.", + lambda: panel and panel.begin_add_highlight_flow(), + ) ), ) add_highlight_btn.grid(row=2, column=0, sticky="ew", padx=5, pady=(0, 5)) @@ -1088,9 +1108,11 @@ def _load_cluster(): temp_remove_btn = ttk.Button( temp_box, text="Remove Selected", - command=_message_action( - "Removing the selected songs from the temporary playlist.", - lambda: panel and panel.remove_selected_from_temp(), + command=_right_panel_action( + _message_action( + "Removing the selected songs from the temporary playlist.", + lambda: panel and panel.remove_selected_from_temp(), + ) ), ) temp_remove_btn.grid(row=3, column=0, sticky="ew", padx=5, pady=(0, 5)) @@ -1098,9 +1120,11 @@ def _load_cluster(): create_playlist_btn = ttk.Button( temp_box, text="Create Playlist", - command=_message_action( - "Saving the current temporary playlist as a new playlist.", - lambda: panel and panel.create_temp_playlist(), + command=_right_panel_action( + _message_action( + "Saving the current temporary playlist as a new playlist.", + lambda: panel and panel.create_temp_playlist(), + ) ), ) create_playlist_btn.grid(row=4, column=0, sticky="ew", padx=5, pady=(0, 5)) @@ -1166,6 +1190,8 @@ def _auto_create_all(): auto_btn = ttk.Button(btn_frame, text="Auto-Create", command=_auto_create_all) auto_btn.pack(side="left", padx=(5, 0)) + ttk.Label(btn_frame, textvariable=banner_var).pack(side="left", padx=(5, 0)) + redo_btn: ttk.Button | None = None if name == "Interactive – HDBSCAN": redo_btn = ttk.Button( From 0a17fb4ba04265a447f575022a4d9592d2d716b2 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 16 Dec 2025 10:50:06 -0500 Subject: [PATCH 088/606] Remove deprecated playlist cluster buttons --- main_gui.py | 80 ----------------------------------------------------- 1 file changed, 80 deletions(-) diff --git a/main_gui.py b/main_gui.py index 6e843ad..96954de 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1134,76 +1134,8 @@ def _load_cluster(): btn_frame = ttk.Frame(container) btn_frame.grid(row=2, column=0, sticky="ew", pady=5) - lasso_var = tk.BooleanVar(value=False) - - lasso_btn = ttk.Checkbutton( - btn_frame, - text="Lasso Mode", - variable=lasso_var, - command=_message_action( - "Toggling lasso mode for manual graph selection.", - lambda: panel and panel.toggle_lasso(), - ), - ) - lasso_btn.pack(side="left") - - ok_btn = ttk.Button( - btn_frame, - text="OK", - command=_message_action( - "Finalizing the current lasso selection into the temp playlist.", - lambda: panel and panel.finalize_lasso(), - ), - ) - ok_btn.pack(side="left", padx=(5, 0)) - - gen_btn = ttk.Button( - btn_frame, - text="Generate Playlist", - command=_message_action( - "Building a playlist from the current selection.", - lambda: panel and panel.create_playlist(), - ), - ) - gen_btn.pack(side="left", padx=(5, 0)) - - def _auto_create_all(): - _set_message("Auto-creating playlists for every available cluster.") - if panel is None or not getattr(panel, "cluster_params", None): - return - method = panel.cluster_params.get("method") - params = { - k: v - for k, v in panel.cluster_params.items() - if k not in {"method", "engine"} - } - engine = panel.cluster_params.get("engine", "librosa") - if not params: - return - app.show_log_tab() - threading.Thread( - target=app._run_cluster_generation, - args=(app.library_path, method, params, engine), - daemon=True, - ).start() - - auto_btn = ttk.Button(btn_frame, text="Auto-Create", command=_auto_create_all) - auto_btn.pack(side="left", padx=(5, 0)) - ttk.Label(btn_frame, textvariable=banner_var).pack(side="left", padx=(5, 0)) - redo_btn: ttk.Button | None = None - if name == "Interactive – HDBSCAN": - redo_btn = ttk.Button( - btn_frame, - text="Redo Values", - command=_message_action( - "Reopen the clustering parameters to tweak HDBSCAN values.", - lambda: panel and panel.open_param_dialog(), - ), - ) - redo_btn.pack(side="left", padx=(5, 0)) - placeholder: ttk.Label | None = None def _sync_cluster_controls(cluster_ready: bool, running: bool) -> None: @@ -1220,16 +1152,9 @@ def _sync_cluster_controls(cluster_ready: bool, running: bool) -> None: add_highlight_btn, temp_remove_btn, create_playlist_btn, - lasso_btn, - ok_btn, - gen_btn, - auto_btn, ): widget.config(state=state) - if redo_btn is not None: - redo_btn.config(state=state) - run_btn.state(["disabled"] if running else ["!disabled"]) if running: @@ -1310,11 +1235,6 @@ def refresh_cluster_panel(): panel.temp_status_var = temp_status_var panel.temp_listbox = temp_listbox panel.temp_remove_btn = temp_remove_btn - panel.lasso_var = lasso_var - panel.lasso_btn = lasso_btn - panel.ok_btn = ok_btn - panel.gen_btn = gen_btn - hover_panel = ttk.Frame(panel, relief="solid", borderwidth=1) art_lbl = tk.Label(hover_panel) art_lbl.pack(side="left") From 197b37f90b5aad22c510d8a61891665eb0c1076b Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 16 Dec 2025 13:18:55 -0500 Subject: [PATCH 089/606] Add bottom banner counter updates --- main_gui.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/main_gui.py b/main_gui.py index 96954de..70a0241 100644 --- a/main_gui.py +++ b/main_gui.py @@ -943,6 +943,11 @@ def generate(): banner_var = tk.StringVar(value="") press_counter = 0 + def _update_banner() -> None: + nonlocal press_counter + press_counter += 1 + banner_var.set(str(press_counter)) + def _set_message(text: str) -> None: message_var.set(text) @@ -958,9 +963,7 @@ def wrapped() -> Any: def _right_panel_action(action: Callable[[], Any] | None) -> Callable[[], Any]: def wrapped() -> Any: - nonlocal press_counter - press_counter += 1 - banner_var.set(str(press_counter)) + _update_banner() if action is not None: return action() From 6636d87a75102f1f6308f1b76464ede9f794942d Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 16 Dec 2025 17:53:54 -0500 Subject: [PATCH 090/606] Revert "Add banner counter helper for right panel actions" --- main_gui.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/main_gui.py b/main_gui.py index 70a0241..96954de 100644 --- a/main_gui.py +++ b/main_gui.py @@ -943,11 +943,6 @@ def generate(): banner_var = tk.StringVar(value="") press_counter = 0 - def _update_banner() -> None: - nonlocal press_counter - press_counter += 1 - banner_var.set(str(press_counter)) - def _set_message(text: str) -> None: message_var.set(text) @@ -963,7 +958,9 @@ def wrapped() -> Any: def _right_panel_action(action: Callable[[], Any] | None) -> Callable[[], Any]: def wrapped() -> Any: - _update_banner() + nonlocal press_counter + press_counter += 1 + banner_var.set(str(press_counter)) if action is not None: return action() From 1845756f2f2037616e11c0ee0e2e0b2922a48ad5 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 16 Dec 2025 17:55:47 -0500 Subject: [PATCH 091/606] Revert "Remove deprecated cluster playlist controls" --- main_gui.py | 80 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/main_gui.py b/main_gui.py index 96954de..6e843ad 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1134,8 +1134,76 @@ def _load_cluster(): btn_frame = ttk.Frame(container) btn_frame.grid(row=2, column=0, sticky="ew", pady=5) + lasso_var = tk.BooleanVar(value=False) + + lasso_btn = ttk.Checkbutton( + btn_frame, + text="Lasso Mode", + variable=lasso_var, + command=_message_action( + "Toggling lasso mode for manual graph selection.", + lambda: panel and panel.toggle_lasso(), + ), + ) + lasso_btn.pack(side="left") + + ok_btn = ttk.Button( + btn_frame, + text="OK", + command=_message_action( + "Finalizing the current lasso selection into the temp playlist.", + lambda: panel and panel.finalize_lasso(), + ), + ) + ok_btn.pack(side="left", padx=(5, 0)) + + gen_btn = ttk.Button( + btn_frame, + text="Generate Playlist", + command=_message_action( + "Building a playlist from the current selection.", + lambda: panel and panel.create_playlist(), + ), + ) + gen_btn.pack(side="left", padx=(5, 0)) + + def _auto_create_all(): + _set_message("Auto-creating playlists for every available cluster.") + if panel is None or not getattr(panel, "cluster_params", None): + return + method = panel.cluster_params.get("method") + params = { + k: v + for k, v in panel.cluster_params.items() + if k not in {"method", "engine"} + } + engine = panel.cluster_params.get("engine", "librosa") + if not params: + return + app.show_log_tab() + threading.Thread( + target=app._run_cluster_generation, + args=(app.library_path, method, params, engine), + daemon=True, + ).start() + + auto_btn = ttk.Button(btn_frame, text="Auto-Create", command=_auto_create_all) + auto_btn.pack(side="left", padx=(5, 0)) + ttk.Label(btn_frame, textvariable=banner_var).pack(side="left", padx=(5, 0)) + redo_btn: ttk.Button | None = None + if name == "Interactive – HDBSCAN": + redo_btn = ttk.Button( + btn_frame, + text="Redo Values", + command=_message_action( + "Reopen the clustering parameters to tweak HDBSCAN values.", + lambda: panel and panel.open_param_dialog(), + ), + ) + redo_btn.pack(side="left", padx=(5, 0)) + placeholder: ttk.Label | None = None def _sync_cluster_controls(cluster_ready: bool, running: bool) -> None: @@ -1152,9 +1220,16 @@ def _sync_cluster_controls(cluster_ready: bool, running: bool) -> None: add_highlight_btn, temp_remove_btn, create_playlist_btn, + lasso_btn, + ok_btn, + gen_btn, + auto_btn, ): widget.config(state=state) + if redo_btn is not None: + redo_btn.config(state=state) + run_btn.state(["disabled"] if running else ["!disabled"]) if running: @@ -1235,6 +1310,11 @@ def refresh_cluster_panel(): panel.temp_status_var = temp_status_var panel.temp_listbox = temp_listbox panel.temp_remove_btn = temp_remove_btn + panel.lasso_var = lasso_var + panel.lasso_btn = lasso_btn + panel.ok_btn = ok_btn + panel.gen_btn = gen_btn + hover_panel = ttk.Frame(panel, relief="solid", borderwidth=1) art_lbl = tk.Label(hover_panel) art_lbl.pack(side="left") From 6549c03032add6260e19f9627359f77876470d9a Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 16 Dec 2025 21:07:41 -0500 Subject: [PATCH 092/606] Simplify clustering UI controls --- cluster_graph_panel.py | 46 +---------------------------- main_gui.py | 67 +----------------------------------------- 2 files changed, 2 insertions(+), 111 deletions(-) diff --git a/cluster_graph_panel.py b/cluster_graph_panel.py index 81fd3f6..0b14460 100644 --- a/cluster_graph_panel.py +++ b/cluster_graph_panel.py @@ -105,8 +105,6 @@ def __init__( # UI widgets created externally will be assigned after instantiation lasso_btn: ttk.Widget - ok_btn: ttk.Button - gen_btn: ttk.Button def _start_clustering(self, params: dict): """Kick off clustering work in a background thread.""" @@ -168,8 +166,6 @@ def _set_lasso_enabled(self, active: bool): if hasattr(self, "lasso_var"): self.lasso_var.set(active) - self.ok_btn.configure(state="disabled") - self.gen_btn.configure(state="disabled") self.canvas.draw_idle() # ─── Hover Metadata Panel ──────────────────────────────────────────────── @@ -253,8 +249,6 @@ def _on_lasso_select(self, verts): if self.temp_remove_btn is not None: text = "Press to Confirm" if self.selected_indices else "Lasso Selection" self.temp_remove_btn.configure(text=text) - if not self.selected_indices: - self.ok_btn.configure(state="disabled") return if self._add_highlight_pending and self.selected_indices: self.selected_tracks = [self.tracks[i] for i in self.selected_indices] @@ -262,40 +256,6 @@ def _on_lasso_select(self, verts): self._add_highlight_pending = False self._set_lasso_enabled(False) return - if self.selected_indices: - self.ok_btn.configure(state="normal") - else: - self.ok_btn.configure(state="disabled") - - def finalize_lasso(self): - """Lock selection and enable playlist creation.""" - if not self.selected_indices: - self.log("\u26a0 No points selected; please try again.") - self.gen_btn.configure(state="disabled") - return - - self.selected_tracks = [self.tracks[i] for i in self.selected_indices] - self.gen_btn.configure(state="normal") - - # ─── Playlist Creation ────────────────────────────────────────────────── - def create_playlist(self): - if not self.selected_tracks: - self.log("\u26a0 No songs selected") - return - - outfile = self._prompt_playlist_destination("CustomLasso") - if not outfile: - self.log("\u26a0 Playlist save cancelled") - return - - try: - write_playlist(self.selected_tracks, outfile) - except Exception as e: # pragma: no cover - simple GUI log - self.log(f"\u2717 Failed to write playlist: {e}") - else: - self.log(f"\u2713 Playlist written: {outfile}") - self._open_folder(os.path.dirname(outfile)) - self.gen_btn.configure(state="disabled") def create_temp_playlist(self): """Write the temporary playlist to disk and open the folder.""" @@ -400,7 +360,7 @@ def _draw_clusters(self, labels): c=point_colors, s=20, ) - self.ax.set_title("Lasso to select & generate playlist") + self.ax.set_title("Lasso to select text") else: self.scatter.set_offsets(self.X2) self.scatter.set_color(point_colors) @@ -552,10 +512,6 @@ def _sync_controls(self): state = "normal" if self._clusters_ready else "disabled" if hasattr(self, "lasso_btn"): self.lasso_btn.configure(state=state) - if not self._clusters_ready: - for attr in ("ok_btn", "gen_btn"): - if hasattr(self, attr): - getattr(self, attr).configure(state="disabled") def refresh_control_states(self): """Public hook to sync controls after external widgets are attached.""" diff --git a/main_gui.py b/main_gui.py index 6e843ad..6227233 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1138,7 +1138,7 @@ def _load_cluster(): lasso_btn = ttk.Checkbutton( btn_frame, - text="Lasso Mode", + text="text", variable=lasso_var, command=_message_action( "Toggling lasso mode for manual graph selection.", @@ -1147,63 +1147,6 @@ def _load_cluster(): ) lasso_btn.pack(side="left") - ok_btn = ttk.Button( - btn_frame, - text="OK", - command=_message_action( - "Finalizing the current lasso selection into the temp playlist.", - lambda: panel and panel.finalize_lasso(), - ), - ) - ok_btn.pack(side="left", padx=(5, 0)) - - gen_btn = ttk.Button( - btn_frame, - text="Generate Playlist", - command=_message_action( - "Building a playlist from the current selection.", - lambda: panel and panel.create_playlist(), - ), - ) - gen_btn.pack(side="left", padx=(5, 0)) - - def _auto_create_all(): - _set_message("Auto-creating playlists for every available cluster.") - if panel is None or not getattr(panel, "cluster_params", None): - return - method = panel.cluster_params.get("method") - params = { - k: v - for k, v in panel.cluster_params.items() - if k not in {"method", "engine"} - } - engine = panel.cluster_params.get("engine", "librosa") - if not params: - return - app.show_log_tab() - threading.Thread( - target=app._run_cluster_generation, - args=(app.library_path, method, params, engine), - daemon=True, - ).start() - - auto_btn = ttk.Button(btn_frame, text="Auto-Create", command=_auto_create_all) - auto_btn.pack(side="left", padx=(5, 0)) - - ttk.Label(btn_frame, textvariable=banner_var).pack(side="left", padx=(5, 0)) - - redo_btn: ttk.Button | None = None - if name == "Interactive – HDBSCAN": - redo_btn = ttk.Button( - btn_frame, - text="Redo Values", - command=_message_action( - "Reopen the clustering parameters to tweak HDBSCAN values.", - lambda: panel and panel.open_param_dialog(), - ), - ) - redo_btn.pack(side="left", padx=(5, 0)) - placeholder: ttk.Label | None = None def _sync_cluster_controls(cluster_ready: bool, running: bool) -> None: @@ -1221,15 +1164,9 @@ def _sync_cluster_controls(cluster_ready: bool, running: bool) -> None: temp_remove_btn, create_playlist_btn, lasso_btn, - ok_btn, - gen_btn, - auto_btn, ): widget.config(state=state) - if redo_btn is not None: - redo_btn.config(state=state) - run_btn.state(["disabled"] if running else ["!disabled"]) if running: @@ -1312,8 +1249,6 @@ def refresh_cluster_panel(): panel.temp_remove_btn = temp_remove_btn panel.lasso_var = lasso_var panel.lasso_btn = lasso_btn - panel.ok_btn = ok_btn - panel.gen_btn = gen_btn hover_panel = ttk.Frame(panel, relief="solid", borderwidth=1) art_lbl = tk.Label(hover_panel) From 08651ec4fc153ab15effbf2cd678543f14de2bd7 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 16 Dec 2025 21:09:38 -0500 Subject: [PATCH 093/606] Add option to use near-max cores for parallel clustering --- clustered_playlists.py | 16 +++++++++-- controllers/cluster_controller.py | 2 ++ main_gui.py | 46 +++++++++++++++++++++++++++---- 3 files changed, 55 insertions(+), 9 deletions(-) diff --git a/clustered_playlists.py b/clustered_playlists.py index 6dfa5cd..910f85f 100644 --- a/clustered_playlists.py +++ b/clustered_playlists.py @@ -129,14 +129,23 @@ def _extract_worker(path: str, engine: AudioFeatureEngine) -> tuple[str, "np.nda return path, zeros, str(exc) -def _extract_features_parallel(tracks, cache, log_callback, engine: AudioFeatureEngine): +def _extract_features_parallel( + tracks, + cache, + log_callback, + engine: AudioFeatureEngine, + use_max_workers: bool = False, +): """Fill ``cache`` for ``tracks`` using a bounded process pool.""" feats: list["np.ndarray"] = [] missing = [p for p in tracks if p not in cache] if missing: - workers = max(1, min(multiprocessing.cpu_count(), 4)) + if use_max_workers: + workers = max(1, multiprocessing.cpu_count() - 1) + else: + workers = max(1, min(multiprocessing.cpu_count(), 4)) log_callback( f"⚙ Parallel feature extraction for {len(missing)} files ({workers} workers)" ) @@ -274,6 +283,7 @@ def generate_clustered_playlists( log_callback=None, engine: str = "serial", feature_engine: AudioFeatureEngine = "librosa", + use_max_workers: bool = False, ) -> None: """Create clustered data for the given tracks without writing playlists.""" if log_callback is None: @@ -316,7 +326,7 @@ def generate_clustered_playlists( if engine_mode == "parallel": feats, updated = _extract_features_parallel( - tracks, cache, log_callback, feature_engine + tracks, cache, log_callback, feature_engine, use_max_workers=use_max_workers ) else: feats, updated = _extract_features_serial( diff --git a/controllers/cluster_controller.py b/controllers/cluster_controller.py index 98cc6f7..1251056 100644 --- a/controllers/cluster_controller.py +++ b/controllers/cluster_controller.py @@ -50,6 +50,7 @@ def cluster_library( folder_filter: dict | None = None, engine: str = "serial", feature_engine: str = "librosa", + use_max_workers: bool = False, ) -> tuple[list[str], list]: """Generate clustered playlists for ``library_path`` and return features.""" @@ -73,5 +74,6 @@ def log(msg: str) -> None: log, engine=engine, feature_engine=feature_engine, + use_max_workers=use_max_workers, ) return tracks, feats diff --git a/main_gui.py b/main_gui.py index 6e843ad..8919fed 100644 --- a/main_gui.py +++ b/main_gui.py @@ -2573,6 +2573,9 @@ def cluster_playlists_dialog(self, method: str = "kmeans"): if engine_default == "librosa": engine_default = "serial" engine_var = tk.StringVar(value=engine_default) + use_max_workers_var = tk.BooleanVar( + value=bool(cluster_cfg.get("use_max_workers", False)) + ) top = ttk.Frame(dlg) top.pack(fill="x", padx=10, pady=(10, 0)) @@ -2621,6 +2624,22 @@ def _update_fields(*args): value="parallel", ).pack(anchor="w", padx=5, pady=(0, 5)) + use_max_workers_chk = ttk.Checkbutton( + engine_frame, + text="Aggressive parallelism (use all cores minus one)", + variable=use_max_workers_var, + ) + use_max_workers_chk.pack(anchor="w", padx=22, pady=(0, 5)) + + def _update_engine_state(*_args): + if engine_var.get() == "parallel": + use_max_workers_chk.state(["!disabled"]) + else: + use_max_workers_chk.state(["disabled"]) + + engine_var.trace_add("write", _update_engine_state) + _update_engine_state() + # KMeans params km_frame = ttk.Frame(params_frame) km_var = tk.StringVar(value=str(cluster_cfg.get("n_clusters", 5))) @@ -2796,12 +2815,16 @@ def generate(): f"{epsilon_var.get()} is not a valid number", ) return - self._start_cluster_playlists(m, params, engine, dlg) + self._start_cluster_playlists( + m, params, engine, dlg, use_max_workers_var.get() + ) ttk.Button(btns, text="Generate", command=generate).pack(side="left", padx=5) ttk.Button(btns, text="Cancel", command=dlg.destroy).pack(side="left", padx=5) - def _start_cluster_playlists(self, method: str, params: dict, engine: str, dlg): + def _start_cluster_playlists( + self, method: str, params: dict, engine: str, dlg, use_max_workers: bool + ): if dlg is not None: dlg.destroy() path = self.require_library() @@ -2812,19 +2835,30 @@ def _start_cluster_playlists(self, method: str, params: dict, engine: str, dlg): self.show_log_tab() threading.Thread( target=self._run_cluster_generation, - args=(path, method, params, engine), + args=(path, method, params, engine, use_max_workers), daemon=True, ).start() def _run_cluster_generation( - self, path: str, method: str, params: dict, engine: str + self, path: str, method: str, params: dict, engine: str, use_max_workers: bool ): try: tracks, feats = cluster_library( - path, method, params, self._log, self.folder_filter, engine + path, + method, + params, + self._log, + self.folder_filter, + engine, + use_max_workers=use_max_workers, ) self.cluster_data = (tracks, feats) - self.cluster_params = {"method": method, "engine": engine, **params} + self.cluster_params = { + "method": method, + "engine": engine, + "use_max_workers": use_max_workers, + **params, + } self.cluster_manager = ClusterComputationManager(tracks, feats, self._log) def done(): From ef450e7494e239a8b699d88da58c2bb8f8edc01a Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 16 Dec 2025 21:20:19 -0500 Subject: [PATCH 094/606] Update cluster UI instruction text --- main_gui.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/main_gui.py b/main_gui.py index 354292e..c28a372 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1138,7 +1138,10 @@ def _load_cluster(): lasso_btn = ttk.Checkbutton( btn_frame, - text="text", + text=( + "To begin, press the \"Run Clusters\" button, select library folders, " + "and run. You'll then see the music visually." + ), variable=lasso_var, command=_message_action( "Toggling lasso mode for manual graph selection.", From f0eff04e91e70c804377cec7e668c3474d4e6123 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 16 Dec 2025 21:30:58 -0500 Subject: [PATCH 095/606] Reuse cached fingerprints when unchanged --- fingerprint_generator.py | 120 ++++++++++++++++++++++++++++----------- 1 file changed, 87 insertions(+), 33 deletions(-) diff --git a/fingerprint_generator.py b/fingerprint_generator.py index 5a7c632..3844bed 100644 --- a/fingerprint_generator.py +++ b/fingerprint_generator.py @@ -1,6 +1,6 @@ import os import sqlite3 -from typing import Callable, Tuple +from typing import Callable, Iterable, List, Tuple import tempfile from pydub import AudioSegment, silence @@ -62,16 +62,48 @@ def compute_fingerprint_for_file(args: Tuple[str, str, bool]) -> Tuple[str, int return path, None, None, str(e) +def _load_cached_entries(conn: sqlite3.Connection) -> dict[str, Tuple[float, int | None, str | None]]: + """Return a mapping of path -> (mtime, duration, fingerprint).""" + + cursor = conn.execute( + "SELECT path, mtime, duration, fingerprint FROM fingerprints" + ) + return {row[0]: (row[1], row[2], row[3]) for row in cursor.fetchall()} + + +def _gather_audio_files(root_path: str | Iterable[str]) -> List[str]: + """Return a list of audio files either from a directory walk or iterable.""" + + if isinstance(root_path, str): + audio_files: List[str] = [] + for dirpath, _, files in os.walk(root_path): + rel_dir = os.path.relpath(dirpath, root_path) + if {"not sorted", "playlists"} & {p.lower() for p in rel_dir.split(os.sep)}: + continue + for fname in files: + ext = os.path.splitext(fname)[1].lower() + if ext in SUPPORTED_EXTS: + audio_files.append(os.path.join(dirpath, fname)) + return audio_files + + return [p for p in root_path if os.path.splitext(p)[1].lower() in SUPPORTED_EXTS] + + def compute_fingerprints_parallel( - root_path: str, + root_path: str | Iterable[str], db_path: str, log_callback: Callable[[str], None] | None = None, progress_callback: Callable[[int, int, str, str], None] | None = None, max_workers: int | None = None, trim_silence: bool = False, phase: str = "A", -) -> None: - """Walk ``root_path`` and compute fingerprints using multiple processes.""" +) -> list[tuple[str, int | None, str | None]]: + """Compute fingerprints and return ``(path, duration, fingerprint)`` results. + + ``root_path`` may be a directory or an iterable of specific file paths. Existing + fingerprints in ``db_path`` are reused when their mtime matches, allowing + subsequent scans to skip unchanged files without sacrificing quality. + """ if log_callback is None: log_callback = print @@ -96,41 +128,55 @@ def progress_callback(current: int, total: int, msg: str, _phase: str) -> None: """ ) - audio_files = [] - idx = 0 - for dirpath, _, files in os.walk(root_path): - rel_dir = os.path.relpath(dirpath, root_path) - if {"not sorted", "playlists"} & {p.lower() for p in rel_dir.split(os.sep)}: - continue - for fname in files: - ext = os.path.splitext(fname)[1].lower() - if ext in SUPPORTED_EXTS: - full = os.path.join(dirpath, fname) - audio_files.append(full) - idx += 1 - progress_callback(idx, 0, os.path.relpath(full, root_path), phase) + results: list[tuple[str, int | None, str | None]] = [] + audio_files = _gather_audio_files(root_path) total = len(audio_files) progress_callback(0, total, "Fingerprinting", phase) - work_items = [(p, db_path, trim_silence) for p in audio_files] - with ProcessPoolExecutor(max_workers=max_workers) as exe: - for idx, (path, duration, fp_hash, err) in enumerate(exe.map(compute_fingerprint_for_file, work_items), start=1): - if err: - log_callback(f" ! Failed fingerprint {path}: {err}") - continue - mtime = os.path.getmtime(ensure_long_path(path)) - conn.execute( - "INSERT OR REPLACE INTO fingerprints (path, mtime, duration, fingerprint) VALUES (?, ?, ?, ?)", - (path, mtime, duration, fp_hash), - ) - log_callback(f"Fingerprinted {path}") - progress_callback(idx, total, path, phase) - if idx % 50 == 0 or idx == total: - log_callback(f" • Fingerprinting {idx}/{total}") + cached = _load_cached_entries(conn) + pending: list[str] = [] + completed = 0 + + for path in audio_files: + mtime = os.path.getmtime(ensure_long_path(path)) + cache_entry = cached.get(path) + if cache_entry and cache_entry[2] is not None and cache_entry[0] == mtime: + duration, fp_hash = cache_entry[1], cache_entry[2] + results.append((path, duration, fp_hash)) + completed += 1 + progress_callback(completed, total, path, phase) + continue + pending.append(path) + + work_items = [(p, db_path, trim_silence) for p in pending] + + if pending: + with ProcessPoolExecutor(max_workers=max_workers) as exe: + for path, duration, fp_hash, err in exe.map( + compute_fingerprint_for_file, work_items + ): + completed += 1 + if err: + log_callback(f" ! Failed fingerprint {path}: {err}") + progress_callback(completed, total, path, phase) + continue + mtime = os.path.getmtime(ensure_long_path(path)) + conn.execute( + "INSERT OR REPLACE INTO fingerprints (path, mtime, duration, fingerprint) VALUES (?, ?, ?, ?)", + (path, mtime, duration, fp_hash), + ) + results.append((path, duration, fp_hash)) + log_callback(f"Fingerprinted {path}") + progress_callback(completed, total, path, phase) + if completed % 50 == 0 or completed == total: + log_callback(f" • Fingerprinting {completed}/{total}") + else: + log_callback("All fingerprints loaded from cache") conn.commit() conn.close() + return results @@ -144,5 +190,13 @@ def compute_fingerprints( phase: str = "A", ) -> None: """Backward-compatible wrapper around :func:`compute_fingerprints_parallel`.""" - compute_fingerprints_parallel(root_path, db_path, log_callback, progress_callback, max_workers, trim_silence, phase) + return compute_fingerprints_parallel( + root_path, + db_path, + log_callback, + progress_callback, + max_workers, + trim_silence, + phase, + ) From 1cd490a6ed60588022926a2bb4fe3b6a27f3032a Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 16 Dec 2025 21:39:14 -0500 Subject: [PATCH 096/606] Revert "Reuse cached fingerprints when unchanged" --- fingerprint_generator.py | 120 +++++++++++---------------------------- 1 file changed, 33 insertions(+), 87 deletions(-) diff --git a/fingerprint_generator.py b/fingerprint_generator.py index 3844bed..5a7c632 100644 --- a/fingerprint_generator.py +++ b/fingerprint_generator.py @@ -1,6 +1,6 @@ import os import sqlite3 -from typing import Callable, Iterable, List, Tuple +from typing import Callable, Tuple import tempfile from pydub import AudioSegment, silence @@ -62,48 +62,16 @@ def compute_fingerprint_for_file(args: Tuple[str, str, bool]) -> Tuple[str, int return path, None, None, str(e) -def _load_cached_entries(conn: sqlite3.Connection) -> dict[str, Tuple[float, int | None, str | None]]: - """Return a mapping of path -> (mtime, duration, fingerprint).""" - - cursor = conn.execute( - "SELECT path, mtime, duration, fingerprint FROM fingerprints" - ) - return {row[0]: (row[1], row[2], row[3]) for row in cursor.fetchall()} - - -def _gather_audio_files(root_path: str | Iterable[str]) -> List[str]: - """Return a list of audio files either from a directory walk or iterable.""" - - if isinstance(root_path, str): - audio_files: List[str] = [] - for dirpath, _, files in os.walk(root_path): - rel_dir = os.path.relpath(dirpath, root_path) - if {"not sorted", "playlists"} & {p.lower() for p in rel_dir.split(os.sep)}: - continue - for fname in files: - ext = os.path.splitext(fname)[1].lower() - if ext in SUPPORTED_EXTS: - audio_files.append(os.path.join(dirpath, fname)) - return audio_files - - return [p for p in root_path if os.path.splitext(p)[1].lower() in SUPPORTED_EXTS] - - def compute_fingerprints_parallel( - root_path: str | Iterable[str], + root_path: str, db_path: str, log_callback: Callable[[str], None] | None = None, progress_callback: Callable[[int, int, str, str], None] | None = None, max_workers: int | None = None, trim_silence: bool = False, phase: str = "A", -) -> list[tuple[str, int | None, str | None]]: - """Compute fingerprints and return ``(path, duration, fingerprint)`` results. - - ``root_path`` may be a directory or an iterable of specific file paths. Existing - fingerprints in ``db_path`` are reused when their mtime matches, allowing - subsequent scans to skip unchanged files without sacrificing quality. - """ +) -> None: + """Walk ``root_path`` and compute fingerprints using multiple processes.""" if log_callback is None: log_callback = print @@ -128,55 +96,41 @@ def progress_callback(current: int, total: int, msg: str, _phase: str) -> None: """ ) - results: list[tuple[str, int | None, str | None]] = [] - audio_files = _gather_audio_files(root_path) + audio_files = [] + idx = 0 + for dirpath, _, files in os.walk(root_path): + rel_dir = os.path.relpath(dirpath, root_path) + if {"not sorted", "playlists"} & {p.lower() for p in rel_dir.split(os.sep)}: + continue + for fname in files: + ext = os.path.splitext(fname)[1].lower() + if ext in SUPPORTED_EXTS: + full = os.path.join(dirpath, fname) + audio_files.append(full) + idx += 1 + progress_callback(idx, 0, os.path.relpath(full, root_path), phase) total = len(audio_files) progress_callback(0, total, "Fingerprinting", phase) + work_items = [(p, db_path, trim_silence) for p in audio_files] - cached = _load_cached_entries(conn) - pending: list[str] = [] - completed = 0 - - for path in audio_files: - mtime = os.path.getmtime(ensure_long_path(path)) - cache_entry = cached.get(path) - if cache_entry and cache_entry[2] is not None and cache_entry[0] == mtime: - duration, fp_hash = cache_entry[1], cache_entry[2] - results.append((path, duration, fp_hash)) - completed += 1 - progress_callback(completed, total, path, phase) - continue - pending.append(path) - - work_items = [(p, db_path, trim_silence) for p in pending] - - if pending: - with ProcessPoolExecutor(max_workers=max_workers) as exe: - for path, duration, fp_hash, err in exe.map( - compute_fingerprint_for_file, work_items - ): - completed += 1 - if err: - log_callback(f" ! Failed fingerprint {path}: {err}") - progress_callback(completed, total, path, phase) - continue - mtime = os.path.getmtime(ensure_long_path(path)) - conn.execute( - "INSERT OR REPLACE INTO fingerprints (path, mtime, duration, fingerprint) VALUES (?, ?, ?, ?)", - (path, mtime, duration, fp_hash), - ) - results.append((path, duration, fp_hash)) - log_callback(f"Fingerprinted {path}") - progress_callback(completed, total, path, phase) - if completed % 50 == 0 or completed == total: - log_callback(f" • Fingerprinting {completed}/{total}") - else: - log_callback("All fingerprints loaded from cache") + with ProcessPoolExecutor(max_workers=max_workers) as exe: + for idx, (path, duration, fp_hash, err) in enumerate(exe.map(compute_fingerprint_for_file, work_items), start=1): + if err: + log_callback(f" ! Failed fingerprint {path}: {err}") + continue + mtime = os.path.getmtime(ensure_long_path(path)) + conn.execute( + "INSERT OR REPLACE INTO fingerprints (path, mtime, duration, fingerprint) VALUES (?, ?, ?, ?)", + (path, mtime, duration, fp_hash), + ) + log_callback(f"Fingerprinted {path}") + progress_callback(idx, total, path, phase) + if idx % 50 == 0 or idx == total: + log_callback(f" • Fingerprinting {idx}/{total}") conn.commit() conn.close() - return results @@ -190,13 +144,5 @@ def compute_fingerprints( phase: str = "A", ) -> None: """Backward-compatible wrapper around :func:`compute_fingerprints_parallel`.""" - return compute_fingerprints_parallel( - root_path, - db_path, - log_callback, - progress_callback, - max_workers, - trim_silence, - phase, - ) + compute_fingerprints_parallel(root_path, db_path, log_callback, progress_callback, max_workers, trim_silence, phase) From 1db33e7cdcd0b3cd24cfa748b89e116d5eda9cb6 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 16 Dec 2025 21:49:00 -0500 Subject: [PATCH 097/606] Add tests for cached fingerprint reuse --- fingerprint_generator.py | 120 ++++++++++++++++++-------- tests/test_fingerprint_generator.py | 126 ++++++++++++++++++++++++++++ 2 files changed, 213 insertions(+), 33 deletions(-) create mode 100644 tests/test_fingerprint_generator.py diff --git a/fingerprint_generator.py b/fingerprint_generator.py index 5a7c632..3844bed 100644 --- a/fingerprint_generator.py +++ b/fingerprint_generator.py @@ -1,6 +1,6 @@ import os import sqlite3 -from typing import Callable, Tuple +from typing import Callable, Iterable, List, Tuple import tempfile from pydub import AudioSegment, silence @@ -62,16 +62,48 @@ def compute_fingerprint_for_file(args: Tuple[str, str, bool]) -> Tuple[str, int return path, None, None, str(e) +def _load_cached_entries(conn: sqlite3.Connection) -> dict[str, Tuple[float, int | None, str | None]]: + """Return a mapping of path -> (mtime, duration, fingerprint).""" + + cursor = conn.execute( + "SELECT path, mtime, duration, fingerprint FROM fingerprints" + ) + return {row[0]: (row[1], row[2], row[3]) for row in cursor.fetchall()} + + +def _gather_audio_files(root_path: str | Iterable[str]) -> List[str]: + """Return a list of audio files either from a directory walk or iterable.""" + + if isinstance(root_path, str): + audio_files: List[str] = [] + for dirpath, _, files in os.walk(root_path): + rel_dir = os.path.relpath(dirpath, root_path) + if {"not sorted", "playlists"} & {p.lower() for p in rel_dir.split(os.sep)}: + continue + for fname in files: + ext = os.path.splitext(fname)[1].lower() + if ext in SUPPORTED_EXTS: + audio_files.append(os.path.join(dirpath, fname)) + return audio_files + + return [p for p in root_path if os.path.splitext(p)[1].lower() in SUPPORTED_EXTS] + + def compute_fingerprints_parallel( - root_path: str, + root_path: str | Iterable[str], db_path: str, log_callback: Callable[[str], None] | None = None, progress_callback: Callable[[int, int, str, str], None] | None = None, max_workers: int | None = None, trim_silence: bool = False, phase: str = "A", -) -> None: - """Walk ``root_path`` and compute fingerprints using multiple processes.""" +) -> list[tuple[str, int | None, str | None]]: + """Compute fingerprints and return ``(path, duration, fingerprint)`` results. + + ``root_path`` may be a directory or an iterable of specific file paths. Existing + fingerprints in ``db_path`` are reused when their mtime matches, allowing + subsequent scans to skip unchanged files without sacrificing quality. + """ if log_callback is None: log_callback = print @@ -96,41 +128,55 @@ def progress_callback(current: int, total: int, msg: str, _phase: str) -> None: """ ) - audio_files = [] - idx = 0 - for dirpath, _, files in os.walk(root_path): - rel_dir = os.path.relpath(dirpath, root_path) - if {"not sorted", "playlists"} & {p.lower() for p in rel_dir.split(os.sep)}: - continue - for fname in files: - ext = os.path.splitext(fname)[1].lower() - if ext in SUPPORTED_EXTS: - full = os.path.join(dirpath, fname) - audio_files.append(full) - idx += 1 - progress_callback(idx, 0, os.path.relpath(full, root_path), phase) + results: list[tuple[str, int | None, str | None]] = [] + audio_files = _gather_audio_files(root_path) total = len(audio_files) progress_callback(0, total, "Fingerprinting", phase) - work_items = [(p, db_path, trim_silence) for p in audio_files] - with ProcessPoolExecutor(max_workers=max_workers) as exe: - for idx, (path, duration, fp_hash, err) in enumerate(exe.map(compute_fingerprint_for_file, work_items), start=1): - if err: - log_callback(f" ! Failed fingerprint {path}: {err}") - continue - mtime = os.path.getmtime(ensure_long_path(path)) - conn.execute( - "INSERT OR REPLACE INTO fingerprints (path, mtime, duration, fingerprint) VALUES (?, ?, ?, ?)", - (path, mtime, duration, fp_hash), - ) - log_callback(f"Fingerprinted {path}") - progress_callback(idx, total, path, phase) - if idx % 50 == 0 or idx == total: - log_callback(f" • Fingerprinting {idx}/{total}") + cached = _load_cached_entries(conn) + pending: list[str] = [] + completed = 0 + + for path in audio_files: + mtime = os.path.getmtime(ensure_long_path(path)) + cache_entry = cached.get(path) + if cache_entry and cache_entry[2] is not None and cache_entry[0] == mtime: + duration, fp_hash = cache_entry[1], cache_entry[2] + results.append((path, duration, fp_hash)) + completed += 1 + progress_callback(completed, total, path, phase) + continue + pending.append(path) + + work_items = [(p, db_path, trim_silence) for p in pending] + + if pending: + with ProcessPoolExecutor(max_workers=max_workers) as exe: + for path, duration, fp_hash, err in exe.map( + compute_fingerprint_for_file, work_items + ): + completed += 1 + if err: + log_callback(f" ! Failed fingerprint {path}: {err}") + progress_callback(completed, total, path, phase) + continue + mtime = os.path.getmtime(ensure_long_path(path)) + conn.execute( + "INSERT OR REPLACE INTO fingerprints (path, mtime, duration, fingerprint) VALUES (?, ?, ?, ?)", + (path, mtime, duration, fp_hash), + ) + results.append((path, duration, fp_hash)) + log_callback(f"Fingerprinted {path}") + progress_callback(completed, total, path, phase) + if completed % 50 == 0 or completed == total: + log_callback(f" • Fingerprinting {completed}/{total}") + else: + log_callback("All fingerprints loaded from cache") conn.commit() conn.close() + return results @@ -144,5 +190,13 @@ def compute_fingerprints( phase: str = "A", ) -> None: """Backward-compatible wrapper around :func:`compute_fingerprints_parallel`.""" - compute_fingerprints_parallel(root_path, db_path, log_callback, progress_callback, max_workers, trim_silence, phase) + return compute_fingerprints_parallel( + root_path, + db_path, + log_callback, + progress_callback, + max_workers, + trim_silence, + phase, + ) diff --git a/tests/test_fingerprint_generator.py b/tests/test_fingerprint_generator.py new file mode 100644 index 0000000..bdef212 --- /dev/null +++ b/tests/test_fingerprint_generator.py @@ -0,0 +1,126 @@ +import os +import sqlite3 +import time +import types +import sys + +# Stub heavy dependencies before importing the module under test +acoustid_stub = types.ModuleType("acoustid") +acoustid_stub.fingerprint_file = lambda path: (0, "stub-fp") +sys.modules["acoustid"] = acoustid_stub + +silence_stub = types.ModuleType("pydub.silence") +silence_stub.detect_nonsilent = lambda *args, **kwargs: [] + +AudioSegment = type("AudioSegment", (), {"from_file": staticmethod(lambda path: None)}) +pydub_stub = types.ModuleType("pydub") +pydub_stub.AudioSegment = AudioSegment +pydub_stub.silence = silence_stub +sys.modules["pydub"] = pydub_stub +sys.modules["pydub.silence"] = silence_stub + +import fingerprint_generator + + +SCHEMA = """ +CREATE TABLE IF NOT EXISTS fingerprints ( + path TEXT PRIMARY KEY, + mtime REAL, + duration INT, + fingerprint TEXT +); +""" + + +def _init_db(db_path: str) -> sqlite3.Connection: + conn = sqlite3.connect(db_path) + conn.execute(SCHEMA) + conn.commit() + return conn + + +def test_cached_fingerprints_reused(tmp_path): + audio = tmp_path / "song.mp3" + audio.write_text("data") + db = tmp_path / "fp.db" + conn = _init_db(db) + mtime = os.path.getmtime(audio) + conn.execute( + "INSERT OR REPLACE INTO fingerprints (path, mtime, duration, fingerprint) VALUES (?, ?, ?, ?)", + (str(audio), mtime, 111, "fp-cache"), + ) + conn.commit() + conn.close() + + logs: list[str] = [] + progress: list[tuple[int, int, str]] = [] + + results = fingerprint_generator.compute_fingerprints_parallel( + [str(audio)], + str(db), + log_callback=logs.append, + progress_callback=lambda c, t, p, _ph: progress.append((c, t, p)), + max_workers=1, + ) + + assert results == [(str(audio), 111, "fp-cache")] + assert any("All fingerprints loaded from cache" in line for line in logs) + assert (1, 1, str(audio)) in progress + + +class DummyExecutor: + def __init__(self, *_, **__): + pass + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def map(self, fn, iterable): + return map(fn, iterable) + + +def test_mtime_change_triggers_recompute(monkeypatch, tmp_path): + audio = tmp_path / "song.mp3" + audio.write_text("old") + db = tmp_path / "fp.db" + conn = _init_db(db) + old_mtime = os.path.getmtime(audio) + conn.execute( + "INSERT OR REPLACE INTO fingerprints (path, mtime, duration, fingerprint) VALUES (?, ?, ?, ?)", + (str(audio), old_mtime, 5, "old-fp"), + ) + conn.commit() + conn.close() + + def fake_compute(args): + path, _db_path, _trim = args + return path, 9, "new-fp", None + + # ensure filesystem mtime advances + time.sleep(0.01) + audio.write_text("new") + + monkeypatch.setattr(fingerprint_generator, "ProcessPoolExecutor", DummyExecutor) + monkeypatch.setattr(fingerprint_generator, "compute_fingerprint_for_file", fake_compute) + + results = fingerprint_generator.compute_fingerprints_parallel( + [str(audio)], + str(db), + log_callback=lambda _msg: None, + progress_callback=lambda *_args, **_kwargs: None, + max_workers=1, + ) + + assert results == [(str(audio), 9, "new-fp")] + conn = sqlite3.connect(db) + row = conn.execute( + "SELECT mtime, duration, fingerprint FROM fingerprints WHERE path=?", + (str(audio),), + ).fetchone() + conn.close() + + assert row[2] == "new-fp" + assert row[0] == os.path.getmtime(audio) From cf143597517a0fb1b2e805f25b8dfe4a979f9ffb Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 16 Dec 2025 21:57:47 -0500 Subject: [PATCH 098/606] Force cluster graphs to redraw after clustering --- cluster_graph_panel.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cluster_graph_panel.py b/cluster_graph_panel.py index 0b14460..d4a1e24 100644 --- a/cluster_graph_panel.py +++ b/cluster_graph_panel.py @@ -475,6 +475,9 @@ def _resize_once() -> None: h = widget.winfo_height() if w <= 1 or h <= 1: return + # Reset cached size so ``on_resize`` always triggers a redraw, + # mimicking a manual window wiggle even if the dimensions are unchanged. + self._last_size = None self.on_resize(w, h) self.after_idle(_resize_once) From a1f6b34395c3a0f4dc2e4ced337d7f327a0ebd42 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 16 Dec 2025 21:58:43 -0500 Subject: [PATCH 099/606] Revert "Force cluster graphs to redraw after clustering" --- cluster_graph_panel.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/cluster_graph_panel.py b/cluster_graph_panel.py index d4a1e24..0b14460 100644 --- a/cluster_graph_panel.py +++ b/cluster_graph_panel.py @@ -475,9 +475,6 @@ def _resize_once() -> None: h = widget.winfo_height() if w <= 1 or h <= 1: return - # Reset cached size so ``on_resize`` always triggers a redraw, - # mimicking a manual window wiggle even if the dimensions are unchanged. - self._last_size = None self.on_resize(w, h) self.after_idle(_resize_once) From e14000014e4c267510d01f0dfa602893d5a1fa6a Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Wed, 17 Dec 2025 19:24:27 -0500 Subject: [PATCH 100/606] Replace sort by genre with genre normalizer --- main_gui.py | 326 ++++++---------------------------------------------- 1 file changed, 35 insertions(+), 291 deletions(-) diff --git a/main_gui.py b/main_gui.py index c28a372..c6f4538 100644 --- a/main_gui.py +++ b/main_gui.py @@ -221,322 +221,66 @@ def km_func(X, p): **extras, } algo_key = "hdbscan" - elif name == "Sort by Genre": + elif name == "Genre Normalizer": lib_var = tk.StringVar(value=app.library_path or "No library selected") - progress_var = tk.StringVar(value="Waiting to start") - playlists_dir = tk.StringVar(value="") - running = tk.BooleanVar(value=False) - total_tracks = 0 - q: queue.Queue = queue.Queue() - cancel_event = threading.Event() - genre_vars: dict[str, tk.BooleanVar] = {} - current_buckets: dict[str, list[str]] = {} - planned_paths: dict[str, str] = {} norm_status = tk.StringVar(value="Select a library to enable normalization.") - norm_btn: ttk.Button | None = None - - def append_log(msg: str) -> None: - log_box.configure(state="normal") - log_box.insert("end", msg + "\n") - log_box.see("end") - log_box.configure(state="disabled") - - def render_stats(stats: dict | None): - for child in stats_rows.winfo_children(): - child.destroy() - genre_vars.clear() - if not stats: - ttk.Label( - stats_rows, - text="No playlists generated yet. Run a sort to preview genres.", - ).pack(anchor="w", padx=5, pady=5) - update_controls() - return - header = ttk.Frame(stats_rows) - header.pack(fill="x", padx=5, pady=(0, 4)) - ttk.Label(header, text="Export", width=8, anchor="w").pack( - side="left", padx=(0, 4) - ) - ttk.Label(header, text="Genre", width=26, anchor="w").pack(side="left") - ttk.Label(header, text="Tracks", width=8).pack(side="left") - ttk.Label(header, text="Playlist", width=20, anchor="w").pack( - side="left", padx=(10, 0) - ) - ttk.Label(header, text="Status", width=12, anchor="w").pack( - side="left", padx=(6, 0) - ) - for genre, info in sorted(stats.items(), key=lambda kv: kv[0].lower()): - row = ttk.Frame(stats_rows) - row.pack(fill="x", padx=5, pady=2) - var = tk.BooleanVar(value=info.get("exported", False) or True) - genre_vars[genre] = var - ttk.Checkbutton(row, variable=var).pack(side="left", padx=(0, 6)) - ttk.Label(row, text=genre, width=26, anchor="w").pack(side="left") - ttk.Label(row, text=str(info.get("count", 0)), width=8).pack( - side="left" - ) - ttk.Label( - row, text=os.path.basename(info.get("playlist", "")), width=20 - ).pack(side="left", padx=(10, 5)) - status_txt = "Exported" if info.get("exported") else "Pending" - ttk.Label(row, text=status_txt, width=12).pack(side="left", padx=(6, 4)) - ttk.Button( - row, - text="Open", - command=lambda p=info.get("playlist", ""): open_path(p), - state="normal" if info.get("exported") else "disabled", - ).pack(side="left") - update_controls() def update_controls(): lib_var.set(app.library_path or "No library selected") - ready = bool(app.library_path) and not running.get() - has_results = bool(current_buckets) - run_btn.config(state="normal" if ready else "disabled") - cancel_btn.config(state="normal" if running.get() else "disabled") - open_btn.config( - state="normal" if playlists_dir.get() or app.library_path else "disabled" - ) - export_btn.config( - state="normal" - if ready and has_results and any(v.get() for v in genre_vars.values()) - else "disabled" - ) - select_all_btn.config(state="normal" if has_results and ready else "disabled") - select_none_btn.config(state="normal" if has_results and ready else "disabled") norm_status.set( os.path.join(app.library_path, ".genre_mapping.json") if app.library_path else "Select a library to enable normalization." ) - if norm_btn: - norm_btn.config( - state="normal" if app.library_path and not running.get() else "disabled" - ) + open_btn.config(state="normal" if app.library_path else "disabled") - def open_path(path: str) -> None: - if not path: - return - try: - if sys.platform == "win32": - os.startfile(path) # type: ignore[attr-defined] - elif sys.platform == "darwin": - subprocess.run(["open", path], check=False) - else: - subprocess.run(["xdg-open", path], check=False) - except Exception as exc: # pragma: no cover - OS interaction - messagebox.showerror("Open File", f"Could not open {path}: {exc}") - - def select_all(): - for var in genre_vars.values(): - var.set(True) - update_controls() - - def select_none(): - for var in genre_vars.values(): - var.set(False) - update_controls() - - def export_selected(): - if running.get() or not current_buckets: - return - selected = [g for g, var in genre_vars.items() if var.get()] - if not selected: - messagebox.showinfo( - "No Genres Selected", - "Select at least one genre to export playlists.", - ) - return - result = playlist_engine.export_genre_playlists( - current_buckets, - app.library_path, - selected_genres=selected, - log_callback=append_log, - planned_paths=planned_paths, - ) - render_stats(result.get("genres")) - append_log(f"Exported {len(selected)} playlist(s).") - open_genre_folder() + intro = ttk.Frame(frame) + intro.pack(fill="both", expand=True, padx=10, pady=10) - def start_run(): - nonlocal total_tracks - if running.get(): - return - path = app.require_library() - if not path: - return - current_buckets.clear() - planned_paths.clear() - genre_vars.clear() - tracks = gather_tracks(path, getattr(app, "folder_filter", None)) - if not tracks: - messagebox.showinfo("No Tracks", "No audio files found in the library.") - return - total_tracks = len(tracks) - playlists_dir.set(os.path.join(path, "Playlists", "Genres")) - cancel_event.clear() - running.set(True) - progress["value"] = 0 - progress["maximum"] = total_tracks - progress_var.set(f"0/{total_tracks} processed") - log_box.configure(state="normal") - log_box.delete("1.0", "end") - log_box.configure(state="disabled") - append_log(f"Found {total_tracks} tracks. Sorting by genre…") - render_stats(None) - update_controls() - - def worker(): - try: - result = playlist_engine.sort_tracks_by_genre( - tracks, - path, - log_callback=lambda m: q.put(("log", m)), - progress_callback=lambda c: q.put(("progress", c)), - cancel_event=cancel_event, - export=False, - ) - q.put(("done", result)) - except Exception as exc: # pragma: no cover - UI surface only - q.put(("error", str(exc))) - - threading.Thread(target=worker, daemon=True).start() - poll_queue() - - def poll_queue(): - nonlocal current_buckets, planned_paths - try: - while True: - tag, payload = q.get_nowait() - if tag == "log": - append_log(payload) - elif tag == "progress": - progress["value"] = payload - progress_var.set(f"{payload}/{total_tracks} processed") - elif tag == "done": - running.set(False) - if payload: - current_buckets = payload.get("buckets", {}) - planned_paths = payload.get("playlist_paths", {}) - render_stats(payload.get("genres")) - update_controls() - elif tag == "error": - running.set(False) - messagebox.showerror("Sort by Genre", payload) - update_controls() - elif tag == "cancelled": - running.set(False) - append_log("Sorting cancelled") - update_controls() - except queue.Empty: - pass - if running.get(): - frame.after(200, poll_queue) + ttk.Label( + intro, + text=( + "Normalize genre labels across your library by generating or " + "updating the .genre_mapping.json file." + ), + wraplength=480, + justify="left", + ).pack(anchor="w", pady=(0, 10)) - def cancel_run(): - cancel_event.set() - q.put(("cancelled", None)) + status_row = ttk.Frame(intro) + status_row.pack(fill="x", pady=(0, 6)) + ttk.Label(status_row, text="Library:", width=10).pack(side="left") + ttk.Label(status_row, textvariable=lib_var).pack(side="left", fill="x", expand=True) - def open_genre_folder(): - path = playlists_dir.get() or ( - os.path.join(app.library_path, "Playlists", "Genres") - if app.library_path - else "" - ) - if not path: - return - os.makedirs(path, exist_ok=True) - open_path(path) - - paned = ttk.Panedwindow(frame, orient="horizontal") - paned.pack(fill="both", expand=True) - - main_area = ttk.Frame(paned) - side_panel = ttk.Frame(paned, width=280) - paned.add(main_area, weight=3) - paned.add(side_panel, weight=2) - - info = ttk.Frame(main_area) - info.pack(fill="x", pady=(0, 6)) - ttk.Label(info, textvariable=lib_var).pack(side="left") - open_btn = ttk.Button(info, text="Open Genre Playlists", command=open_genre_folder) - open_btn.pack(side="right", padx=(5, 0)) - run_btn = ttk.Button(info, text="Sort", command=start_run) - run_btn.pack(side="right", padx=(5, 0)) - cancel_btn = ttk.Button(info, text="Cancel", command=cancel_run, state="disabled") - cancel_btn.pack(side="right") - - prog_frame = ttk.Frame(main_area) - prog_frame.pack(fill="x", pady=(0, 6)) - progress = ttk.Progressbar(prog_frame, mode="determinate") - progress.pack(fill="x") - ttk.Label(prog_frame, textvariable=progress_var).pack(anchor="w") - - selection_bar = ttk.Frame(main_area) - selection_bar.pack(fill="x", pady=(0, 4)) - select_all_btn = ttk.Button( - selection_bar, text="Select All", command=select_all, state="disabled" - ) - select_all_btn.pack(side="left") - select_none_btn = ttk.Button( - selection_bar, text="Select None", command=select_none, state="disabled" - ) - select_none_btn.pack(side="left", padx=(5, 0)) - export_btn = ttk.Button( - selection_bar, - text="Export Selected Playlists", - command=export_selected, - state="disabled", - ) - export_btn.pack(side="right") - - stats_container = ttk.Frame(main_area) - stats_container.pack(fill="both", pady=(0, 6), expand=True) - stats_canvas = tk.Canvas(stats_container, height=220) - stats_scroll = ttk.Scrollbar(stats_container, orient="vertical", command=stats_canvas.yview) - stats_rows = ttk.Frame(stats_canvas) - stats_rows.bind( - "", - lambda e: stats_canvas.configure(scrollregion=stats_canvas.bbox("all")), + map_row = ttk.Frame(intro) + map_row.pack(fill="x", pady=(0, 12)) + ttk.Label(map_row, text="Mapping file:", width=10).pack(side="left") + ttk.Label(map_row, textvariable=norm_status, wraplength=360).pack( + side="left", fill="x", expand=True ) - stats_canvas.create_window((0, 0), window=stats_rows, anchor="nw") - stats_canvas.configure(yscrollcommand=stats_scroll.set) - stats_canvas.pack(side="left", fill="both", expand=True) - stats_scroll.pack(side="right", fill="y") - render_stats(None) - - ttk.Label(main_area, text="Activity Log:").pack(anchor="w") - log_box = ScrolledText(main_area, height=10, wrap="word", state="disabled") - log_box.pack(fill="both", expand=True) - - # ── Genre Normalizer Side Panel ───────────────────────────────── - normalizer_box = ttk.LabelFrame(side_panel, text="Genre Normalizer") - normalizer_box.pack(fill="both", expand=True, padx=(10, 0)) ttk.Label( - normalizer_box, + intro, text=( - "Create or edit a mapping file before exporting playlists. " - "The mapping will normalize genres during sorting." + "Use the Genre Normalization Assistant to scan raw genres, copy " + "the suggested prompt, and paste or edit your mapping JSON." ), - wraplength=240, + wraplength=480, justify="left", - ).pack(fill="x", padx=8, pady=(8, 6)) - - ttk.Label(normalizer_box, text="Mapping file:").pack(anchor="w", padx=8) - ttk.Label(normalizer_box, textvariable=norm_status, wraplength=240).pack( - fill="x", padx=8 - ) + ).pack(anchor="w", pady=(0, 10)) - norm_btn = ttk.Button( - normalizer_box, + open_btn = ttk.Button( + intro, text="Open Genre Normalizer", command=lambda: app._open_genre_normalizer(), ) - norm_btn.pack(pady=8, padx=8, anchor="e") + open_btn.pack(anchor="e") - update_controls() + def refresh_panel(): + update_controls() + frame.refresh_cluster_panel = refresh_panel + update_controls() elif name == "Tempo/Energy Buckets": lib_var = tk.StringVar(value=app.library_path or "No library selected") dep_status = tk.StringVar() @@ -1719,7 +1463,7 @@ def build_ui(self): for name in [ "Interactive – KMeans", "Interactive – HDBSCAN", - "Sort by Genre", + "Genre Normalizer", "Tempo/Energy Buckets", "Auto-DJ", ]: From 4576444b5712427fedc2b08ae27b575a839c183b Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Wed, 17 Dec 2025 19:27:18 -0500 Subject: [PATCH 101/606] Fix hiding playlist plugin panels when switching tools --- main_gui.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/main_gui.py b/main_gui.py index c28a372..325ec07 100644 --- a/main_gui.py +++ b/main_gui.py @@ -2129,9 +2129,11 @@ def on_plugin_select(self, event): return logging.info("[perf] tool switch -> %s", sel) - if self.active_plugin and self.active_plugin in self.plugin_views: - logging.info("[perf] hide panel %s", self.active_plugin) - self.plugin_views[self.active_plugin].pack_forget() + # Ensure any previously displayed plugin panels are hidden before showing the + # newly selected tool. This avoids orphaned UIs sticking around when + # switching between tools (e.g., the Tempo/Energy Buckets view). + for panel in self.plugin_views.values(): + panel.pack_forget() panel = self.plugin_views.get(sel) if panel is None: From d414a1d3ceae390d40890ad8c7826d2ece3c0981 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Wed, 17 Dec 2025 19:31:26 -0500 Subject: [PATCH 102/606] Inline genre normalizer panel --- main_gui.py | 242 +++++++++++++++++++++++++++------------------------- 1 file changed, 127 insertions(+), 115 deletions(-) diff --git a/main_gui.py b/main_gui.py index 279907d..abfd01e 100644 --- a/main_gui.py +++ b/main_gui.py @@ -224,6 +224,8 @@ def km_func(X, p): elif name == "Genre Normalizer": lib_var = tk.StringVar(value=app.library_path or "No library selected") norm_status = tk.StringVar(value="Select a library to enable normalization.") + scan_status = tk.StringVar(value="Scan genres to populate the assistant.") + scanning = tk.BooleanVar(value=False) def update_controls(): lib_var.set(app.library_path or "No library selected") @@ -232,10 +234,11 @@ def update_controls(): if app.library_path else "Select a library to enable normalization." ) - open_btn.config(state="normal" if app.library_path else "disabled") + scan_btn.config(state="normal" if app.library_path else "disabled") + apply_btn.config(state="normal" if app.library_path else "disabled") intro = ttk.Frame(frame) - intro.pack(fill="both", expand=True, padx=10, pady=10) + intro.pack(fill="x", padx=10, pady=10) ttk.Label( intro, @@ -249,38 +252,131 @@ def update_controls(): status_row = ttk.Frame(intro) status_row.pack(fill="x", pady=(0, 6)) - ttk.Label(status_row, text="Library:", width=10).pack(side="left") - ttk.Label(status_row, textvariable=lib_var).pack(side="left", fill="x", expand=True) + ttk.Label(status_row, text="Library:", width=12).pack(side="left") + ttk.Label(status_row, textvariable=lib_var).pack( + side="left", fill="x", expand=True + ) map_row = ttk.Frame(intro) - map_row.pack(fill="x", pady=(0, 12)) - ttk.Label(map_row, text="Mapping file:", width=10).pack(side="left") + map_row.pack(fill="x", pady=(0, 6)) + ttk.Label(map_row, text="Mapping file:", width=12).pack(side="left") ttk.Label(map_row, textvariable=norm_status, wraplength=360).pack( side="left", fill="x", expand=True ) - ttk.Label( - intro, - text=( - "Use the Genre Normalization Assistant to scan raw genres, copy " - "the suggested prompt, and paste or edit your mapping JSON." - ), - wraplength=480, - justify="left", - ).pack(anchor="w", pady=(0, 10)) - - open_btn = ttk.Button( - intro, - text="Open Genre Normalizer", - command=lambda: app._open_genre_normalizer(), + control_row = ttk.Frame(intro) + control_row.pack(fill="x", pady=(4, 0)) + scan_btn = ttk.Button(control_row, text="Scan Genres") + scan_btn.pack(side="left") + ttk.Label(control_row, textvariable=scan_status).pack( + side="left", padx=(8, 0) + ) + prog = ttk.Progressbar( + control_row, orient="horizontal", length=160, mode="determinate" ) - open_btn.pack(anchor="e") + prog.pack(side="right") + + prompt_box = ttk.LabelFrame(frame, text="LLM Prompt Template") + prompt_box.pack(fill="both", expand=False, padx=10, pady=(0, 10)) + app.text_prompt = ScrolledText(prompt_box, height=8, wrap="word") + app.text_prompt.pack(fill="both", expand=True, padx=8, pady=6) + app.text_prompt.insert("1.0", PROMPT_TEMPLATE.strip()) + app.text_prompt.configure(state="disabled") + ttk.Button( + prompt_box, + text="Copy Prompt", + command=lambda: app.clipboard_append(PROMPT_TEMPLATE.strip()), + ).pack(anchor="e", padx=8, pady=(0, 6)) + + raw_box = ttk.LabelFrame(frame, text="Raw Genre List") + raw_box.pack(fill="both", expand=True, padx=10, pady=(0, 10)) + app.text_raw = ScrolledText(raw_box, width=50, height=12) + app.text_raw.pack(fill="both", expand=True, padx=8, pady=6) + app.text_raw.insert("1.0", "Scan the library to populate raw genres.") + app.text_raw.configure(state="disabled") + ttk.Button( + raw_box, + text="Copy Raw List", + command=lambda: app.clipboard_append("\n".join(getattr(app, "raw_genre_list", []))), + ).pack(anchor="e", padx=8, pady=(0, 6)) + + map_box = ttk.LabelFrame(frame, text="Paste JSON Mapping Here") + map_box.pack(fill="both", expand=True, padx=10, pady=(0, 10)) + app.text_map = ScrolledText(map_box, width=50, height=10) + app.text_map.pack(fill="both", expand=True, padx=8, pady=6) + + btn_frame = ttk.Frame(map_box) + btn_frame.pack(fill="x", padx=8, pady=(0, 6)) + apply_btn = ttk.Button(btn_frame, text="Apply Mapping", command=app.apply_mapping) + apply_btn.pack(side="right") + + def populate_raw_genres(genres: list[str]): + app.text_raw.configure(state="normal") + app.text_raw.delete("1.0", "end") + if genres: + app.text_raw.insert("1.0", "\n".join(genres)) + else: + app.text_raw.insert("1.0", "No genres found.") + app.text_raw.configure(state="disabled") + + def load_existing_mapping(): + if not app.library_path: + app.text_map.delete("1.0", "end") + return + try: + with open(norm_status.get(), "r", encoding="utf-8") as f: + existing = json.load(f) + except Exception: + existing = {} + app.text_map.delete("1.0", "end") + app.text_map.insert("1.0", json.dumps(existing, indent=2)) + + def on_progress(idx, total): + try: + prog["value"] = idx + prog["maximum"] = max(total, 1) + except Exception: + pass + + def scan_task(folder: str): + try: + app.raw_genre_list = scan_raw_genres(folder, on_progress) + finally: + app.after(0, lambda: finish_scan(folder)) + + def finish_scan(folder: str): + scanning.set(False) + scan_btn.config(state="normal") + scan_status.set(f"Found {len(getattr(app, 'raw_genre_list', []))} genres.") + populate_raw_genres(getattr(app, "raw_genre_list", [])) + prog["value"] = prog["maximum"] + app.mapping_path = os.path.join(folder, ".genre_mapping.json") + + def start_scan(): + folder = app.require_library() + if not folder or scanning.get(): + return + files = discover_files(folder) + if not files: + messagebox.showinfo("No audio files", "No supported audio found.") + return + scanning.set(True) + scan_status.set("Scanning genres…") + prog["value"] = 0 + prog["maximum"] = len(files) + scan_btn.config(state="disabled") + threading.Thread(target=scan_task, args=(folder,), daemon=True).start() + + scan_btn.config(command=start_scan) def refresh_panel(): update_controls() + if getattr(app, "raw_genre_list", None): + populate_raw_genres(app.raw_genre_list) + load_existing_mapping() frame.refresh_cluster_panel = refresh_panel - update_controls() + refresh_panel() elif name == "Tempo/Energy Buckets": lib_var = tk.StringVar(value=app.library_path or "No library selected") dep_status = tk.StringVar() @@ -3033,98 +3129,18 @@ def _open_dup_debug_window(self) -> None: self.dup_debug_win = win self.dup_text = text - def _open_genre_normalizer(self, _show_dialog: bool = False): - """Open a dialog to assist with genre normalization.""" - folder = self.require_library() - if not folder: - return - - # Always refresh mapping path - self.mapping_path = os.path.join(folder, ".genre_mapping.json") - - if not _show_dialog: - # ── Show progress bar and run raw-only scan ── - files = discover_files(folder) - if not files: - messagebox.showinfo("No audio files", "No supported audio found.") - return - - prog_win = tk.Toplevel(self) - prog_win.title("Scanning Genres…") - prog_bar = ttk.Progressbar( - prog_win, orient="horizontal", length=300, mode="determinate" + def apply_mapping(self): + """Persist mapping JSON from text box and apply normalization.""" + if not hasattr(self, "text_map"): + messagebox.showwarning( + "No Mapping Editor", "Open the Genre Normalizer to edit mappings." ) - prog_bar.pack(padx=20, pady=20) - prog_bar["maximum"] = len(files) - - def on_progress(idx, total): - prog_bar["value"] = idx - prog_win.update_idletasks() - - def scan_task(): - self.raw_genre_list = scan_raw_genres(folder, on_progress) - prog_win.destroy() - # reopen dialog to show panels - self.after(0, lambda: self._open_genre_normalizer(True)) - - threading.Thread(target=scan_task, daemon=True).start() return - # ── Now _show_dialog == True: build the three-panel dialog ── - win = tk.Toplevel(self) - win.title("Genre Normalization Assistant") - win.grab_set() - - # 1) LLM Prompt - tk.Label(win, text="LLM Prompt Template:").pack( - anchor="w", padx=10, pady=(10, 0) - ) - self.text_prompt = ScrolledText(win, height=8, wrap="word") - self.text_prompt.pack(fill="both", padx=10) - self.text_prompt.insert("1.0", PROMPT_TEMPLATE.strip()) - self.text_prompt.configure(state="disabled") - ttk.Button( - win, - text="Copy Prompt", - command=lambda: self.clipboard_append(PROMPT_TEMPLATE.strip()), - ).pack(anchor="e", padx=10, pady=(0, 10)) - - # 2) Raw Genre List - tk.Label(win, text="Raw Genre List:").pack(anchor="w", padx=10) - self.text_raw = ScrolledText(win, width=50, height=15) - self.text_raw.pack(fill="both", padx=10, pady=(0, 10)) - self.text_raw.insert("1.0", "\n".join(self.raw_genre_list)) - self.text_raw.configure(state="disabled") - ttk.Button( - win, - text="Copy Raw List", - command=lambda: self.clipboard_append("\n".join(self.raw_genre_list)), - ).pack(anchor="e", padx=10, pady=(0, 10)) - - # 3) Mapping JSON Input - tk.Label(win, text="Paste JSON Mapping Here:").pack(anchor="w", padx=10) - self.text_map = ScrolledText(win, width=50, height=10) - self.text_map.pack(fill="both", padx=10, pady=(0, 10)) - # pre-load existing mapping - try: - with open(self.mapping_path, "r", encoding="utf-8") as f: - existing = json.load(f) - except Exception: - existing = {} - self.text_map.insert("1.0", json.dumps(existing, indent=2)) - - # Buttons: Apply Mapping & Close - btn_frame = ttk.Frame(win) - btn_frame.pack(fill="x", padx=10, pady=(0, 10)) - ttk.Button(btn_frame, text="Apply Mapping", command=self.apply_mapping).pack( - side="right" - ) - ttk.Button(btn_frame, text="Close", command=win.destroy).pack( - side="right", padx=(0, 5) - ) + if not getattr(self, "mapping_path", None): + messagebox.showwarning("No Library", "Select a library before applying.") + return - def apply_mapping(self): - """Persist mapping JSON from text box and apply normalization.""" try: with open(self.mapping_path, "r", encoding="utf-8") as f: existing_map = json.load(f) @@ -3162,15 +3178,11 @@ def apply_mapping(self): if hasattr(self, "filtered_records") and hasattr(self, "_prop_tv"): self._render_table(self.filtered_records) - # Confirm success and close the normalization dialog + # Confirm success and keep the editor open messagebox.showinfo( "Mapping Applied", "Your genre mapping has been successfully saved and applied to the library.", ) - try: - self.text_map.winfo_toplevel().destroy() - except Exception: - pass def reset_tagfix_log(self): initial = self.library_path or load_last_path() From dfe760f3199fd2efdeeefa59f05c28f92c94fb75 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Wed, 17 Dec 2025 19:38:26 -0500 Subject: [PATCH 103/606] Fix genre normalizer panel creation --- main_gui.py | 1 + 1 file changed, 1 insertion(+) diff --git a/main_gui.py b/main_gui.py index abfd01e..75657ab 100644 --- a/main_gui.py +++ b/main_gui.py @@ -377,6 +377,7 @@ def refresh_panel(): frame.refresh_cluster_panel = refresh_panel refresh_panel() + return frame elif name == "Tempo/Energy Buckets": lib_var = tk.StringVar(value=app.library_path or "No library selected") dep_status = tk.StringVar() From 429d792cec16a31728ec5a22776d11c1cc3a9458 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Wed, 17 Dec 2025 19:44:48 -0500 Subject: [PATCH 104/606] Add load playlist button to player tab --- main_gui.py | 50 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/main_gui.py b/main_gui.py index 75657ab..45fbe9d 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1806,15 +1806,18 @@ def build_ui(self): art_panel = ttk.Frame(player_content) art_panel.pack(side="right", fill="y", padx=(10, 0)) - self.player_art_label = ttk.Label(art_panel) - self.player_art_label.pack(fill="x", pady=(0, 6)) self.player_art_caption = ttk.Label( art_panel, text="Select a track to view album art", wraplength=220, justify="center", ) - self.player_art_caption.pack(fill="x") + self.player_art_caption.pack(fill="x", pady=(0, 6)) + self.player_art_label = ttk.Label(art_panel) + self.player_art_label.pack(fill="x") + ttk.Button( + art_panel, text="Load Playlist", command=self._player_load_playlist + ).pack(fill="x", pady=(6, 6)) self._update_player_art(None) playlist_box = ttk.LabelFrame(art_panel, text="Playlist Builder") @@ -3515,6 +3518,47 @@ def _player_clear_temp_playlist(self) -> None: return self._player_set_temp_playlist([]) + def _player_load_playlist(self) -> None: + playlists_dir = os.path.join(self.library_path, "Playlists") + initial_dir = ( + playlists_dir + if os.path.isdir(playlists_dir) + else self.library_path + if self.library_path + else os.getcwd() + ) + chosen = filedialog.askopenfilename( + title="Load Playlist", + initialdir=initial_dir, + defaultextension=".m3u", + filetypes=[("M3U Playlist", "*.m3u"), ("All Files", "*.*")], + ) + if not chosen: + return + + try: + with open(chosen, "r", encoding="utf-8") as f: + entries = [line.strip() for line in f if line.strip()] + except Exception as exc: + messagebox.showerror("Load Playlist", f"Could not read playlist: {exc}") + return + + base_dir = os.path.dirname(chosen) + resolved: list[str] = [] + for entry in entries: + candidate = entry if os.path.isabs(entry) else os.path.join(base_dir, entry) + candidate = os.path.normpath(candidate) + if os.path.exists(candidate): + resolved.append(candidate) + + if not resolved: + messagebox.showinfo( + "Load Playlist", "No valid tracks found in the selected playlist." + ) + return + + self._player_set_temp_playlist(resolved) + def _player_save_temp_playlist(self) -> None: if not self.player_temp_playlist: messagebox.showinfo("Playlist Builder", "Add songs before saving a playlist.") From 75208d8f8d33ed7e28753c58f9faeac883d68983 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Wed, 17 Dec 2025 19:47:11 -0500 Subject: [PATCH 105/606] Improve Auto-DJ controls and cancellation --- main_gui.py | 173 ++++++++++++++++++++++++++++++++++++++++----- playlist_engine.py | 10 +++ 2 files changed, 166 insertions(+), 17 deletions(-) diff --git a/main_gui.py b/main_gui.py index 75657ab..264e54f 100644 --- a/main_gui.py +++ b/main_gui.py @@ -698,6 +698,11 @@ def poll_queue(): count_var = tk.StringVar(value="20") engine_var = getattr(app, "feature_engine_var", tk.StringVar(value="librosa")) app.feature_engine_var = engine_var + playlist_name_var = tk.StringVar(value="Auto DJ Mix") + playlist_file_var = tk.StringVar(value="autodj.m3u") + progress_var = tk.StringVar(value="Waiting to start") + running = tk.BooleanVar(value=False) + cancel_event = threading.Event() def resolve_engine() -> str: engine = engine_var.get() @@ -726,7 +731,42 @@ def browse(): if f: sel.set(f) + def append_log(msg: str) -> None: + def _append() -> None: + app._log(msg) + log_box.configure(state="normal") + log_box.insert("end", msg + "\n") + log_box.see("end") + log_box.configure(state="disabled") + + app.after(0, _append) + + def set_progress(value: int, maximum: int, message: str | None = None) -> None: + def _update() -> None: + progress["maximum"] = max(1, maximum) + progress["value"] = value + if message: + progress_var.set(message) + + app.after(0, _update) + + def reset_ui(status: str) -> None: + running.set(False) + cancel_event.clear() + progress["value"] = 0 + progress_var.set(status) + generate_btn.config(state="normal") + cancel_btn.config(state="disabled") + + def cancel_generation() -> None: + cancel_event.set() + append_log("⚠ Cancelling Auto-DJ run…") + progress_var.set("Cancelling…") + cancel_btn.config(state="disabled") + def generate(): + if running.get(): + return path = app.require_library() if not path or not sel.get(): messagebox.showerror("Error", "Select a library and track") @@ -736,24 +776,123 @@ def generate(): except ValueError: messagebox.showerror("Error", "Invalid count") return + + playlist_name = playlist_name_var.get().strip() or "Auto DJ Mix" + filename = playlist_file_var.get().strip() or "autodj.m3u" + if not filename.lower().endswith(".m3u"): + filename += ".m3u" + app.show_log_tab() - tracks = gather_tracks(path) - engine = resolve_engine() - order = autodj_playlist( - sel.get(), tracks, n, log_callback=app._log, engine=engine - ) - outfile = os.path.join(path, "Playlists", "autodj.m3u") - write_playlist(order, outfile) - messagebox.showinfo("Playlist", f"Written to {outfile}") - open_path(os.path.dirname(outfile)) - - row = ttk.Frame(frame) - row.pack(padx=10, pady=10) - tk.Entry(row, textvariable=sel, width=40).pack(side="left") - ttk.Button(row, text="Browse", command=browse).pack(side="left", padx=5) - ttk.Entry(row, textvariable=count_var, width=5).pack(side="left") - ttk.Label(row, text="songs").pack(side="left") - ttk.Button(row, text="Generate", command=generate).pack(side="left", padx=5) + running.set(True) + cancel_event.clear() + generate_btn.config(state="disabled") + cancel_btn.config(state="normal") + progress_var.set("Gathering tracks…") + log_box.configure(state="normal") + log_box.delete("1.0", "end") + log_box.configure(state="disabled") + + def worker() -> None: + try: + append_log("→ Gathering tracks from library…") + tracks = gather_tracks(path) + if cancel_event.is_set(): + raise IndexCancelled() + if not tracks: + raise RuntimeError("No tracks found in library") + if sel.get() not in tracks: + raise RuntimeError("Selected start track is not in the library") + + engine = resolve_engine() + + def progress_cb(current: int, total: int, message: str | None = None): + set_progress(current, total, message) + + append_log( + f"→ Generating {playlist_name} with {n} songs using {engine} features" + ) + order = autodj_playlist( + sel.get(), + tracks, + n, + log_callback=append_log, + engine=engine, + progress_callback=progress_cb, + cancel_event=cancel_event, + ) + if cancel_event.is_set(): + raise IndexCancelled() + + outfile = os.path.join(path, "Playlists", filename) + write_playlist(order, outfile) + append_log(f"✓ {playlist_name} written to {outfile}") + set_progress(len(order), max(len(order), 1), "Complete") + app.after( + 0, + lambda: messagebox.showinfo( + "Playlist", f"{playlist_name} written to {outfile}" + ), + ) + app.after(0, lambda: open_path(os.path.dirname(outfile))) + except IndexCancelled: + append_log("✖ Auto-DJ cancelled.") + app.after(0, lambda: progress_var.set("Cancelled")) + except Exception as exc: + append_log(f"✖ Error during Auto-DJ: {exc}") + app.after(0, lambda: messagebox.showerror("Playlist", str(exc))) + finally: + app.after(0, lambda: reset_ui("Ready")) + + threading.Thread(target=worker, daemon=True).start() + + row = ttk.LabelFrame(frame, text="Auto-DJ Settings") + row.pack(fill="x", padx=10, pady=(10, 5)) + path_row = ttk.Frame(row) + path_row.pack(fill="x", pady=5) + ttk.Label(path_row, text="Start track:").pack(side="left") + tk.Entry(path_row, textvariable=sel, width=40).pack(side="left", padx=(5, 0)) + ttk.Button(path_row, text="Browse", command=browse).pack(side="left", padx=5) + + controls_row = ttk.Frame(row) + controls_row.pack(fill="x", pady=5) + ttk.Label(controls_row, text="Songs:").pack(side="left") + ttk.Entry(controls_row, textvariable=count_var, width=5).pack(side="left", padx=(5, 10)) + ttk.Label(controls_row, text="Playlist name:").pack(side="left") + ttk.Entry(controls_row, textvariable=playlist_name_var, width=20).pack( + side="left", padx=(5, 10) + ) + ttk.Label(controls_row, text="File name:").pack(side="left") + ttk.Entry(controls_row, textvariable=playlist_file_var, width=18).pack( + side="left", padx=(5, 10) + ) + ttk.Label(controls_row, text="Engine:").pack(side="left") + ttk.Combobox( + controls_row, + textvariable=engine_var, + values=["librosa", "essentia"], + width=10, + state="readonly", + ).pack(side="left") + + action_row = ttk.Frame(row) + action_row.pack(fill="x", pady=(5, 0)) + generate_btn = ttk.Button(action_row, text="Generate", command=generate) + generate_btn.pack(side="left") + cancel_btn = ttk.Button( + action_row, text="Cancel", command=cancel_generation, state="disabled" + ) + cancel_btn.pack(side="left", padx=5) + + progress = ttk.Progressbar(frame, mode="determinate") + progress.pack(fill="x", padx=10, pady=(5, 0)) + ttk.Label(frame, textvariable=progress_var).pack(anchor="w", padx=12) + + log_group = ttk.LabelFrame(frame, text="Auto-DJ Log") + log_group.pack(fill="both", expand=True, padx=10, pady=(5, 10)) + log_box = ScrolledText(log_group, height=8, state="disabled") + log_box.pack(fill="both", expand=True, padx=5, pady=5) + + reset_ui("Ready") return frame else: ttk.Label(frame, text=f"{name} panel coming soon…").pack(padx=10, pady=10) diff --git a/playlist_engine.py b/playlist_engine.py index c7ceea1..7ef562f 100644 --- a/playlist_engine.py +++ b/playlist_engine.py @@ -47,6 +47,7 @@ class FLAC(_DummyAudio): MonoLoader = RhythmExtractor2013 = None # type: ignore from playlist_generator import write_playlist, DEFAULT_EXTS +from indexer_control import IndexCancelled def categorize_tempo(bpm: float) -> str: @@ -434,13 +435,21 @@ def autodj_playlist( feature_cache=None, log_callback=None, engine: TempoEngine = "librosa", + progress_callback=None, + cancel_event=None, ) -> list[str]: if log_callback is None: log_callback = lambda m: None + if progress_callback is None: + progress_callback = lambda _c, _t, _m=None: None feature_cache = feature_cache or {} order = [start_track] remaining = [t for t in tracks if t != start_track] + total = min(n, len(tracks)) + progress_callback(1, total, "Seed track selected") while remaining and len(order) < n: + if cancel_event is not None and cancel_event.is_set(): + raise IndexCancelled() cur_feat = _get_feat(order[-1], feature_cache, log_callback, engine) next_track = min( remaining, @@ -450,4 +459,5 @@ def autodj_playlist( ) order.append(next_track) remaining.remove(next_track) + progress_callback(len(order), total, f"Added track {len(order)} of {total}") return order From 85c0d6fb05761b4aa75a4d565381d259bb6277e3 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Wed, 17 Dec 2025 19:53:12 -0500 Subject: [PATCH 106/606] Adjust player art header alignment --- main_gui.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main_gui.py b/main_gui.py index e1d6fc4..830fcde 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1890,7 +1890,7 @@ def build_ui(self): self.notebook.add(self.player_tab, text="Player") player_controls = ttk.Frame(self.player_tab) - player_controls.pack(fill="x", padx=10, pady=(10, 5)) + player_controls.pack(fill="x", padx=10, pady=(10, 0)) ttk.Label(player_controls, textvariable=self.player_status_var).pack( side="left" ) @@ -1944,7 +1944,7 @@ def build_ui(self): self.player_tree.bind("<>", self._on_player_selection_change) art_panel = ttk.Frame(player_content) - art_panel.pack(side="right", fill="y", padx=(10, 0)) + art_panel.pack(side="right", padx=(10, 0), anchor="n") self.player_art_caption = ttk.Label( art_panel, text="Select a track to view album art", From c7f8c3631bb4858f11c9b832e4a39bda1b5ea035 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Wed, 17 Dec 2025 19:53:54 -0500 Subject: [PATCH 107/606] Revert "Adjust player tab album art header alignment" --- main_gui.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main_gui.py b/main_gui.py index 830fcde..e1d6fc4 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1890,7 +1890,7 @@ def build_ui(self): self.notebook.add(self.player_tab, text="Player") player_controls = ttk.Frame(self.player_tab) - player_controls.pack(fill="x", padx=10, pady=(10, 0)) + player_controls.pack(fill="x", padx=10, pady=(10, 5)) ttk.Label(player_controls, textvariable=self.player_status_var).pack( side="left" ) @@ -1944,7 +1944,7 @@ def build_ui(self): self.player_tree.bind("<>", self._on_player_selection_change) art_panel = ttk.Frame(player_content) - art_panel.pack(side="right", padx=(10, 0), anchor="n") + art_panel.pack(side="right", fill="y", padx=(10, 0)) self.player_art_caption = ttk.Label( art_panel, text="Select a track to view album art", From 8336be2bae0e439c8abdb60dfc0d29bb18814434 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Wed, 17 Dec 2025 20:03:52 -0500 Subject: [PATCH 108/606] Add split playlist preview to player tab --- main_gui.py | 248 +++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 218 insertions(+), 30 deletions(-) diff --git a/main_gui.py b/main_gui.py index e1d6fc4..2958fe7 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1488,6 +1488,12 @@ def __init__(self): ) self.player_playlist_listbox: tk.Listbox | None = None self.player_view_label: str | None = None + self.player_playlist_mode = False + self.player_playlist_rows: list[dict[str, str]] = [] + self.player_playlist_tree_paths: dict[str, str] = {} + self.player_playlist_tree_rows: dict[str, dict[str, str]] = {} + self.player_playlist_tree: ttk.Treeview | None = None + self.player_playlist_frame: ttk.Frame | None = None # assume ffmpeg is available without performing checks self.ffmpeg_available = True @@ -1911,37 +1917,12 @@ def build_ui(self): player_content = ttk.Frame(self.player_tab) player_content.pack(fill="both", expand=True, padx=10, pady=(0, 10)) - player_table = ttk.Frame(player_content) - player_table.pack(side="left", fill="both", expand=True) - p_vsb = ttk.Scrollbar(player_table, orient="vertical") - p_vsb.pack(side="right", fill="y") - p_hsb = ttk.Scrollbar(player_table, orient="horizontal") - p_hsb.pack(side="bottom", fill="x") - - cols = ("Title", "Artist", "Album", "Length", "Play") - self.player_tree = ttk.Treeview( - player_table, - columns=cols, - show="headings", - yscrollcommand=p_vsb.set, - xscrollcommand=p_hsb.set, + self.player_table_region = ttk.Frame(player_content) + self.player_table_region.pack(side="left", fill="both", expand=True) + self.player_library_frame = self._create_player_library_table( + self.player_table_region ) - p_vsb.config(command=self.player_tree.yview) - p_hsb.config(command=self.player_tree.xview) - self.player_tree.pack(fill="both", expand=True) - - widths = {"Title": 220, "Artist": 140, "Album": 160, "Length": 70, "Play": 50} - for c in cols: - self.player_tree.heading(c, text=c) - self.player_tree.column( - c, - width=widths.get(c, 100), - anchor="center" if c in {"Length", "Play"} else "w", - stretch=c != "Play", - ) - - self.player_tree.bind("", self._on_player_tree_click) - self.player_tree.bind("<>", self._on_player_selection_change) + self.player_library_frame.pack(fill="both", expand=True) art_panel = ttk.Frame(player_content) art_panel.pack(side="right", fill="y", padx=(10, 0)) @@ -1954,6 +1935,10 @@ def build_ui(self): self.player_art_caption.pack(fill="x", pady=(0, 6)) self.player_art_label = ttk.Label(art_panel) self.player_art_label.pack(fill="x") + self.player_playlist_toggle_btn = ttk.Button( + art_panel, text="Add Playlist", command=self._toggle_player_playlist_view + ) + self.player_playlist_toggle_btn.pack(fill="x", pady=(0, 6)) ttk.Button( art_panel, text="Load Playlist", command=self._player_load_playlist ).pack(fill="x", pady=(6, 6)) @@ -3863,6 +3848,209 @@ def do_cancel() -> None: top.wait_window() return result["ok"], skip_var.get() + def _create_player_library_table(self, parent: tk.Widget) -> ttk.Frame: + table = ttk.Frame(parent) + p_vsb = ttk.Scrollbar(table, orient="vertical") + p_vsb.pack(side="right", fill="y") + p_hsb = ttk.Scrollbar(table, orient="horizontal") + p_hsb.pack(side="bottom", fill="x") + + cols = ("Title", "Artist", "Album", "Length", "Play") + self.player_tree = ttk.Treeview( + table, + columns=cols, + show="headings", + yscrollcommand=p_vsb.set, + xscrollcommand=p_hsb.set, + ) + p_vsb.config(command=self.player_tree.yview) + p_hsb.config(command=self.player_tree.xview) + self.player_tree.pack(fill="both", expand=True) + + widths = {"Title": 220, "Artist": 140, "Album": 160, "Length": 70, "Play": 50} + for c in cols: + self.player_tree.heading(c, text=c) + self.player_tree.column( + c, + width=widths.get(c, 100), + anchor="center" if c in {"Length", "Play"} else "w", + stretch=c != "Play", + ) + + self.player_tree.bind("", self._on_player_tree_click) + self.player_tree.bind("<>", self._on_player_selection_change) + return table + + def _create_playlist_table(self, parent: tk.Widget) -> ttk.Treeview: + frame = ttk.Frame(parent) + frame.pack(fill="both", expand=True) + p_vsb = ttk.Scrollbar(frame, orient="vertical") + p_vsb.pack(side="right", fill="y") + p_hsb = ttk.Scrollbar(frame, orient="horizontal") + p_hsb.pack(side="bottom", fill="x") + + cols = ("Title", "Artist", "Play") + tree = ttk.Treeview( + frame, + columns=cols, + show="headings", + yscrollcommand=p_vsb.set, + xscrollcommand=p_hsb.set, + ) + p_vsb.config(command=tree.yview) + p_hsb.config(command=tree.xview) + tree.pack(fill="both", expand=True) + + widths = {"Title": 220, "Artist": 160, "Play": 50} + for c in cols: + tree.heading(c, text=c) + tree.column( + c, + width=widths.get(c, 100), + anchor="center" if c == "Play" else "w", + stretch=c != "Play", + ) + + tree.bind("", self._on_playlist_tree_click) + tree.bind("<>", self._on_playlist_selection_change) + return tree + + def _toggle_player_playlist_view(self) -> None: + if self.player_playlist_mode: + self._unload_player_playlist_view() + return + + tracks = self._prompt_playlist_tracks() + if not tracks: + return + self.player_playlist_rows = self._prepare_player_rows(tracks) + self._enter_player_playlist_view() + + def _prompt_playlist_tracks(self) -> list[str] | None: + playlists_dir = os.path.join(self.library_path, "Playlists") + initial_dir = ( + playlists_dir + if os.path.isdir(playlists_dir) + else self.library_path + if self.library_path + else os.getcwd() + ) + chosen = filedialog.askopenfilename( + title="Add Playlist", + initialdir=initial_dir, + defaultextension=".m3u", + filetypes=[("M3U Playlist", "*.m3u"), ("All Files", "*.*")], + ) + if not chosen: + return None + + try: + with open(chosen, "r", encoding="utf-8") as f: + entries = [line.strip() for line in f if line.strip()] + except Exception as exc: + messagebox.showerror("Playlist", f"Could not read playlist: {exc}") + return None + + base_dir = os.path.dirname(chosen) + resolved: list[str] = [] + for entry in entries: + candidate = entry if os.path.isabs(entry) else os.path.join(base_dir, entry) + candidate = os.path.normpath(candidate) + if os.path.exists(candidate): + resolved.append(candidate) + + if not resolved: + messagebox.showinfo( + "Playlist", "No valid tracks found in the selected playlist." + ) + return None + return resolved + + def _enter_player_playlist_view(self) -> None: + self.player_playlist_mode = True + self.player_playlist_toggle_btn.config(text="Unload Playlist") + self.player_library_frame.pack_forget() + self.player_library_frame.pack( + side="left", fill="both", expand=True, padx=(0, 10), pady=(0, 0) + ) + + self.player_playlist_frame = ttk.LabelFrame( + self.player_table_region, text="Playlist Preview" + ) + self.player_playlist_frame.pack(side="left", fill="both", expand=True) + self.player_playlist_tree = self._create_playlist_table( + self.player_playlist_frame + ) + self._render_playlist_rows(self.player_playlist_rows) + + def _unload_player_playlist_view(self) -> None: + self.player_playlist_mode = False + self.player_playlist_toggle_btn.config(text="Add Playlist") + self.player_playlist_rows = [] + self._clear_playlist_table() + if getattr(self, "player_playlist_frame", None): + self.player_playlist_frame.destroy() + self.player_playlist_frame = None + self.player_library_frame.pack_forget() + self.player_library_frame.pack(fill="both", expand=True) + self._update_player_art(None) + + def _render_playlist_rows(self, rows: list[dict[str, str]]) -> None: + self._clear_playlist_table() + if not self.player_playlist_tree: + return + play_icon = "▶" if self.preview_backend_available else "—" + for row in rows: + item = self.player_playlist_tree.insert( + "", + "end", + values=( + row.get("title", ""), + row.get("artist", ""), + play_icon, + ), + ) + path = row.get("path", "") + self.player_playlist_tree_paths[item] = path + self.player_playlist_tree_rows[item] = row + + def _clear_playlist_table(self) -> None: + if self.player_playlist_tree: + for row in self.player_playlist_tree.get_children(): + self.player_playlist_tree.delete(row) + self.player_playlist_tree_paths.clear() + self.player_playlist_tree_rows.clear() + + def _on_playlist_tree_click(self, event) -> None: + if not self.player_playlist_tree: + return + region = self.player_playlist_tree.identify_region(event.x, event.y) + if region != "cell": + return + col = self.player_playlist_tree.identify_column(event.x) + if col != "#3": + return + item = self.player_playlist_tree.identify_row(event.y) + if not item: + return + path = self.player_playlist_tree_paths.get(item) + if not path: + return + row = self.player_playlist_tree_rows.get(item, {}) + self._update_player_art(path, title=row.get("title"), artist=row.get("artist")) + self._play_preview(path, duration_ms=30000, player_item=None) + + def _on_playlist_selection_change(self, _event) -> None: + if not self.player_playlist_tree: + return + selection = self.player_playlist_tree.selection() + if not selection: + return + item = selection[0] + path = self.player_playlist_tree_paths.get(item) + row = self.player_playlist_tree_rows.get(item, {}) + self._update_player_art(path, title=row.get("title"), artist=row.get("artist")) + # ── Player Tab Helpers ───────────────────────────────────────────── def _clear_player_table(self) -> None: if not hasattr(self, "player_tree"): From 9accac3cbf764958185a6cc67640ae0525a326a8 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Wed, 17 Dec 2025 20:11:47 -0500 Subject: [PATCH 109/606] Include playlist preview selection when adding to builder --- main_gui.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/main_gui.py b/main_gui.py index 2958fe7..532eb2a 100644 --- a/main_gui.py +++ b/main_gui.py @@ -3588,12 +3588,27 @@ def _update_player_art( def _get_selected_player_paths(self) -> list[str]: if not hasattr(self, "player_tree"): return [] + + selections: list[str] = [] + + # Include selections from the main library table. selection = self.player_tree.selection() - return [ - self.player_tree_paths[item] - for item in selection - if item in self.player_tree_paths - ] + selections.extend( + [self.player_tree_paths[item] for item in selection if item in self.player_tree_paths] + ) + + # Include selections from the optional playlist preview table when visible. + if getattr(self, "player_playlist_tree", None): + pl_selection = self.player_playlist_tree.selection() + selections.extend( + [ + self.player_playlist_tree_paths[item] + for item in pl_selection + if item in self.player_playlist_tree_paths + ] + ) + + return selections def _player_set_temp_playlist(self, tracks: list[str]) -> None: self.player_temp_playlist = list(dict.fromkeys(tracks)) From 37849e62f636f43d073058d852f6465cbfa569ee Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Wed, 17 Dec 2025 20:32:41 -0500 Subject: [PATCH 110/606] Add Genre Normalizer initialization --- controllers/normalize_controller.py | 13 ++- main_gui.py | 128 ++++++++++++++++++++++++++++ 2 files changed, 139 insertions(+), 2 deletions(-) diff --git a/controllers/normalize_controller.py b/controllers/normalize_controller.py index ef24ef7..ac2501a 100644 --- a/controllers/normalize_controller.py +++ b/controllers/normalize_controller.py @@ -46,8 +46,17 @@ def save_mapping(folder: str, mapping: Dict[str, str]) -> str: def normalize_genres(genres: list[str], mapping: Dict[str, str]) -> list[str]: - """Return list of genres normalized via mapping.""" - return [mapping.get(g, g) for g in genres] + """Return list of genres normalized via mapping (supports list values).""" + normalized: list[str] = [] + for g in genres: + mapped = mapping.get(g, g) + if isinstance(mapped, list): + normalized.extend(mapped) + elif mapped is None: + continue + else: + normalized.append(mapped) + return normalized def get_raw_genres(records): diff --git a/main_gui.py b/main_gui.py index 532eb2a..0cb6d22 100644 --- a/main_gui.py +++ b/main_gui.py @@ -3,6 +3,7 @@ import sys import logging import webbrowser +import re if sys.platform == "win32": try: @@ -305,10 +306,19 @@ def update_controls(): app.text_map = ScrolledText(map_box, width=50, height=10) app.text_map.pack(fill="both", expand=True, padx=8, pady=6) + init_status = tk.StringVar( + value="Paste JSON and initialize to rewrite library genres." + ) + btn_frame = ttk.Frame(map_box) btn_frame.pack(fill="x", padx=8, pady=(0, 6)) + init_btn = ttk.Button(btn_frame, text="Initialize Normalizer") + init_btn.pack(side="left") apply_btn = ttk.Button(btn_frame, text="Apply Mapping", command=app.apply_mapping) apply_btn.pack(side="right") + ttk.Label(btn_frame, textvariable=init_status).pack( + side="left", padx=8, expand=True, fill="x" + ) def populate_raw_genres(genres: list[str]): app.text_raw.configure(state="normal") @@ -367,7 +377,56 @@ def start_scan(): scan_btn.config(state="disabled") threading.Thread(target=scan_task, args=(folder,), daemon=True).start() + def finish_init(changed: int, total: int): + init_status.set(f"Rewrote genres for {changed} of {total} files.") + init_btn.config(state="normal") + apply_btn.config(state="normal" if app.library_path else "disabled") + scan_btn.config(state="normal" if app.library_path else "disabled") + if getattr(app, "raw_genre_list", None): + populate_raw_genres(app.raw_genre_list) + + def start_init(): + if not app.library_path: + messagebox.showwarning("No Library", "Select a library to initialize.") + return + raw_json = app.text_map.get("1.0", "end").strip() + if not raw_json: + messagebox.showwarning( + "No JSON Provided", "Paste JSON mapping from the assistant first." + ) + return + try: + mapping = json.loads(raw_json) + except json.JSONDecodeError as exc: + messagebox.showerror("Invalid JSON", str(exc)) + return + + init_status.set("Initializing and rewriting genres…") + init_btn.config(state="disabled") + apply_btn.config(state="disabled") + scan_btn.config(state="disabled") + prog["value"] = 0 + + def worker(): + try: + changed, total = app.initialize_genre_normalizer( + mapping, on_progress + ) + except Exception as exc: + app.after( + 0, + lambda: messagebox.showerror( + "Initialization Failed", str(exc) + ), + ) + app.after(0, lambda: finish_init(0, 0)) + return + app.after(0, lambda: finish_init(changed, total)) + + threading.Thread(target=worker, daemon=True).start() + scan_btn.config(command=start_scan) + init_btn.config(command=start_init) def refresh_panel(): update_controls() @@ -3244,6 +3303,75 @@ def _tagfix_debug(self, msg: str): widget.insert("end", msg + "\n") widget.see("end") + def initialize_genre_normalizer(self, mapping: dict, progress_callback=None): + """Persist mapping JSON and rewrite genre tags across the library.""" + if not self.library_path or not getattr(self, "mapping_path", None): + raise ValueError("Library must be selected before initializing the normalizer.") + + normalized_map: dict[str, list[str]] = {} + for raw, value in mapping.items(): + if value is None: + continue + key = str(raw).strip() + if not key: + continue + if isinstance(value, list): + cleaned = [str(v).strip() for v in value if str(v).strip()] + else: + cleaned = [str(value).strip()] if str(value).strip() else [] + if cleaned: + normalized_map[key] = cleaned + + os.makedirs(os.path.dirname(self.mapping_path), exist_ok=True) + with open(self.mapping_path, "w", encoding="utf-8") as f: + json.dump(normalized_map, f, indent=2) + self.genre_mapping = normalized_map + + files = discover_files(self.library_path) + total = len(files) + if progress_callback: + progress_callback(0, total) + + changed = 0 + for idx, path in enumerate(files, start=1): + if progress_callback: + progress_callback(idx, total) + try: + audio = MutagenFile(path, easy=True) + except Exception: + continue + if not audio: + continue + + existing_genres = audio.get("genre", []) or [] + rewritten: list[str] = [] + seen: set[str] = set() + for entry in existing_genres: + parts = re.split(r"[;,/]", entry) + for part in parts: + part = part.strip() + if not part: + continue + mapped = normalized_map.get(part, [part]) + values = mapped if isinstance(mapped, list) else [mapped] + for m in values: + val = str(m).strip() + if val and val not in seen: + seen.add(val) + rewritten.append(val) + + if not rewritten or sorted(rewritten) == sorted(existing_genres): + continue + + try: + audio["genre"] = rewritten + audio.save() + changed += 1 + except Exception: + continue + + return changed, total + def _open_dup_debug_window(self) -> None: if ( getattr(self, "dup_debug_win", None) From 21735f7ab5e4a8cabb760836209a5cea11425908 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Wed, 17 Dec 2025 20:34:07 -0500 Subject: [PATCH 111/606] Add playlist artwork helper --- main_gui.py | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/main_gui.py b/main_gui.py index 532eb2a..2c0bb77 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1544,6 +1544,9 @@ def build_ui(self): label="Generate Library Index…", command=lambda: generate_index(self.require_library()), ) + tools_menu.add_command( + label="Playlist Artwork", command=self.open_playlist_artwork_folder + ) tools_menu.add_separator() tools_menu.add_command(label="Reset Tag-Fix Log", command=self.reset_tagfix_log) menubar.add_cascade(label="Tools", menu=tools_menu) @@ -2176,6 +2179,48 @@ def require_library(self): return None return self.library_path + def _open_path(self, path: str) -> None: + if not path: + return + try: + if sys.platform == "win32": + os.startfile(path) # type: ignore[attr-defined] + elif sys.platform == "darwin": + subprocess.run(["open", path], check=False) + else: + subprocess.run(["xdg-open", path], check=False) + except Exception as exc: # pragma: no cover - OS interaction + messagebox.showerror("Open Folder", f"Could not open {path}: {exc}") + + def open_playlist_artwork_folder(self) -> None: + """Ensure the playlist artwork folder exists and open it for the user.""" + + library = self.require_library() + if not library: + return + + playlists_dir = os.path.join(library, "Playlists") + artwork_dir = os.path.join(playlists_dir, "artwork") + + try: + os.makedirs(artwork_dir, exist_ok=True) + except Exception as exc: + messagebox.showerror( + "Playlist Artwork", + f"Could not create the artwork folder:\n{exc}", + ) + self._log(f"✘ Failed to prepare playlist artwork folder: {exc}") + return + + info = ( + "Playlist artwork folder is ready.\n\n" + f"Location:\n{artwork_dir}\n\n" + "Drop cover images here to accompany your playlists." + ) + messagebox.showinfo("Playlist Artwork", info) + self._log(f"🎨 Playlist artwork folder ready at {artwork_dir}") + self._open_path(artwork_dir) + def _set_status(self, text: str) -> None: self._full_status = text short = textwrap.shorten(text, width=60, placeholder="…") From 1f8b7178105b0d4fe71a64ac3f3be2a6aa41d40d Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Wed, 17 Dec 2025 20:40:19 -0500 Subject: [PATCH 112/606] Add apply-to-songs control to genre normalizer --- main_gui.py | 67 +++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 63 insertions(+), 4 deletions(-) diff --git a/main_gui.py b/main_gui.py index fab8651..cdb679c 100644 --- a/main_gui.py +++ b/main_gui.py @@ -236,6 +236,7 @@ def update_controls(): else "Select a library to enable normalization." ) scan_btn.config(state="normal" if app.library_path else "disabled") + save_btn.config(state="normal" if app.library_path else "disabled") apply_btn.config(state="normal" if app.library_path else "disabled") intro = ttk.Frame(frame) @@ -309,16 +310,22 @@ def update_controls(): init_status = tk.StringVar( value="Paste JSON and initialize to rewrite library genres." ) + apply_status = tk.StringVar( + value="Apply a saved or pasted mapping to embedded genres." + ) btn_frame = ttk.Frame(map_box) btn_frame.pack(fill="x", padx=8, pady=(0, 6)) init_btn = ttk.Button(btn_frame, text="Initialize Normalizer") init_btn.pack(side="left") - apply_btn = ttk.Button(btn_frame, text="Apply Mapping", command=app.apply_mapping) + save_btn = ttk.Button(btn_frame, text="Save Mapping", command=app.apply_mapping) + save_btn.pack(side="left", padx=(6, 0)) + apply_btn = ttk.Button(btn_frame, text="Apply To Songs") apply_btn.pack(side="right") - ttk.Label(btn_frame, textvariable=init_status).pack( - side="left", padx=8, expand=True, fill="x" - ) + status_col = ttk.Frame(btn_frame) + status_col.pack(side="left", padx=8, expand=True, fill="x") + ttk.Label(status_col, textvariable=init_status).pack(anchor="w") + ttk.Label(status_col, textvariable=apply_status).pack(anchor="w") def populate_raw_genres(genres: list[str]): app.text_raw.configure(state="normal") @@ -380,6 +387,7 @@ def start_scan(): def finish_init(changed: int, total: int): init_status.set(f"Rewrote genres for {changed} of {total} files.") init_btn.config(state="normal") + save_btn.config(state="normal" if app.library_path else "disabled") apply_btn.config(state="normal" if app.library_path else "disabled") scan_btn.config(state="normal" if app.library_path else "disabled") if getattr(app, "raw_genre_list", None): @@ -403,6 +411,7 @@ def start_init(): init_status.set("Initializing and rewriting genres…") init_btn.config(state="disabled") + save_btn.config(state="disabled") apply_btn.config(state="disabled") scan_btn.config(state="disabled") prog["value"] = 0 @@ -428,6 +437,56 @@ def worker(): scan_btn.config(command=start_scan) init_btn.config(command=start_init) + def finish_apply(changed: int, total: int): + apply_status.set(f"Applied mapping to {changed} of {total} files.") + init_btn.config(state="normal" if app.library_path else "disabled") + save_btn.config(state="normal" if app.library_path else "disabled") + apply_btn.config(state="normal" if app.library_path else "disabled") + scan_btn.config(state="normal" if app.library_path else "disabled") + if getattr(app, "raw_genre_list", None): + populate_raw_genres(app.raw_genre_list) + + def start_apply(): + if not app.library_path: + messagebox.showwarning("No Library", "Select a library to apply mappings.") + return + raw_json = app.text_map.get("1.0", "end").strip() + if not raw_json: + messagebox.showwarning( + "No JSON Provided", "Paste JSON mapping from the assistant first." + ) + return + try: + mapping = json.loads(raw_json) + except json.JSONDecodeError as exc: + messagebox.showerror("Invalid JSON", str(exc)) + return + + apply_status.set("Applying mapping to songs…") + init_btn.config(state="disabled") + save_btn.config(state="disabled") + apply_btn.config(state="disabled") + scan_btn.config(state="disabled") + prog["value"] = 0 + + def worker(): + try: + changed, total = app.initialize_genre_normalizer( + mapping, on_progress + ) + except Exception as exc: + app.after( + 0, + lambda: messagebox.showerror("Apply Failed", str(exc)), + ) + app.after(0, lambda: finish_apply(0, 0)) + return + app.after(0, lambda: finish_apply(changed, total)) + + threading.Thread(target=worker, daemon=True).start() + + apply_btn.config(command=start_apply) + def refresh_panel(): update_controls() if getattr(app, "raw_genre_list", None): From 0340fbfc5e8bf3242577db7e3af84ecc4a16b6f1 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Wed, 17 Dec 2025 21:00:17 -0500 Subject: [PATCH 113/606] Relocate genre normalizer apply controls --- main_gui.py | 70 +++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 65 insertions(+), 5 deletions(-) diff --git a/main_gui.py b/main_gui.py index fab8651..9b58e9d 100644 --- a/main_gui.py +++ b/main_gui.py @@ -236,6 +236,7 @@ def update_controls(): else "Select a library to enable normalization." ) scan_btn.config(state="normal" if app.library_path else "disabled") + save_btn.config(state="normal" if app.library_path else "disabled") apply_btn.config(state="normal" if app.library_path else "disabled") intro = ttk.Frame(frame) @@ -309,16 +310,23 @@ def update_controls(): init_status = tk.StringVar( value="Paste JSON and initialize to rewrite library genres." ) + apply_status = tk.StringVar( + value="Apply a saved or pasted mapping to embedded genres." + ) btn_frame = ttk.Frame(map_box) btn_frame.pack(fill="x", padx=8, pady=(0, 6)) init_btn = ttk.Button(btn_frame, text="Initialize Normalizer") init_btn.pack(side="left") - apply_btn = ttk.Button(btn_frame, text="Apply Mapping", command=app.apply_mapping) - apply_btn.pack(side="right") - ttk.Label(btn_frame, textvariable=init_status).pack( - side="left", padx=8, expand=True, fill="x" - ) + save_btn = ttk.Button(btn_frame, text="Save Mapping", command=app.apply_mapping) + save_btn.pack(side="left", padx=(6, 0)) + apply_btn = ttk.Button(btn_frame, text="Apply To Songs") + apply_btn.pack(side="left", padx=(6, 0)) + + status_col = ttk.Frame(map_box) + status_col.pack(fill="x", padx=8, pady=(0, 6)) + ttk.Label(status_col, textvariable=init_status).pack(anchor="w") + ttk.Label(status_col, textvariable=apply_status).pack(anchor="w") def populate_raw_genres(genres: list[str]): app.text_raw.configure(state="normal") @@ -380,6 +388,7 @@ def start_scan(): def finish_init(changed: int, total: int): init_status.set(f"Rewrote genres for {changed} of {total} files.") init_btn.config(state="normal") + save_btn.config(state="normal" if app.library_path else "disabled") apply_btn.config(state="normal" if app.library_path else "disabled") scan_btn.config(state="normal" if app.library_path else "disabled") if getattr(app, "raw_genre_list", None): @@ -403,6 +412,7 @@ def start_init(): init_status.set("Initializing and rewriting genres…") init_btn.config(state="disabled") + save_btn.config(state="disabled") apply_btn.config(state="disabled") scan_btn.config(state="disabled") prog["value"] = 0 @@ -428,6 +438,56 @@ def worker(): scan_btn.config(command=start_scan) init_btn.config(command=start_init) + def finish_apply(changed: int, total: int): + apply_status.set(f"Applied mapping to {changed} of {total} files.") + init_btn.config(state="normal" if app.library_path else "disabled") + save_btn.config(state="normal" if app.library_path else "disabled") + apply_btn.config(state="normal" if app.library_path else "disabled") + scan_btn.config(state="normal" if app.library_path else "disabled") + if getattr(app, "raw_genre_list", None): + populate_raw_genres(app.raw_genre_list) + + def start_apply(): + if not app.library_path: + messagebox.showwarning("No Library", "Select a library to apply mappings.") + return + raw_json = app.text_map.get("1.0", "end").strip() + if not raw_json: + messagebox.showwarning( + "No JSON Provided", "Paste JSON mapping from the assistant first." + ) + return + try: + mapping = json.loads(raw_json) + except json.JSONDecodeError as exc: + messagebox.showerror("Invalid JSON", str(exc)) + return + + apply_status.set("Applying mapping to songs…") + init_btn.config(state="disabled") + save_btn.config(state="disabled") + apply_btn.config(state="disabled") + scan_btn.config(state="disabled") + prog["value"] = 0 + + def worker(): + try: + changed, total = app.initialize_genre_normalizer( + mapping, on_progress + ) + except Exception as exc: + app.after( + 0, + lambda: messagebox.showerror("Apply Failed", str(exc)), + ) + app.after(0, lambda: finish_apply(0, 0)) + return + app.after(0, lambda: finish_apply(changed, total)) + + threading.Thread(target=worker, daemon=True).start() + + apply_btn.config(command=start_apply) + def refresh_panel(): update_controls() if getattr(app, "raw_genre_list", None): From 65013b99d9827ac58b72c1a50741a189e9a30a1d Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Wed, 17 Dec 2025 21:04:52 -0500 Subject: [PATCH 114/606] Move genre normalizer actions near top of panel --- main_gui.py | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/main_gui.py b/main_gui.py index cdb679c..b05ff08 100644 --- a/main_gui.py +++ b/main_gui.py @@ -266,6 +266,27 @@ def update_controls(): side="left", fill="x", expand=True ) + init_status = tk.StringVar( + value="Paste JSON and initialize to rewrite library genres." + ) + apply_status = tk.StringVar( + value="Apply a saved or pasted mapping to embedded genres." + ) + + action_bar = ttk.Frame(frame) + action_bar.pack(fill="x", padx=10, pady=(6, 6)) + init_btn = ttk.Button(action_bar, text="Initialize Normalizer") + init_btn.pack(side="left") + save_btn = ttk.Button(action_bar, text="Save Mapping", command=app.apply_mapping) + save_btn.pack(side="left", padx=(6, 0)) + apply_btn = ttk.Button(action_bar, text="Apply To Songs") + apply_btn.pack(side="left", padx=(6, 0)) + + status_col = ttk.Frame(action_bar) + status_col.pack(side="left", padx=12, expand=True, fill="x") + ttk.Label(status_col, textvariable=init_status).pack(anchor="w") + ttk.Label(status_col, textvariable=apply_status).pack(anchor="w") + control_row = ttk.Frame(intro) control_row.pack(fill="x", pady=(4, 0)) scan_btn = ttk.Button(control_row, text="Scan Genres") @@ -307,26 +328,6 @@ def update_controls(): app.text_map = ScrolledText(map_box, width=50, height=10) app.text_map.pack(fill="both", expand=True, padx=8, pady=6) - init_status = tk.StringVar( - value="Paste JSON and initialize to rewrite library genres." - ) - apply_status = tk.StringVar( - value="Apply a saved or pasted mapping to embedded genres." - ) - - btn_frame = ttk.Frame(map_box) - btn_frame.pack(fill="x", padx=8, pady=(0, 6)) - init_btn = ttk.Button(btn_frame, text="Initialize Normalizer") - init_btn.pack(side="left") - save_btn = ttk.Button(btn_frame, text="Save Mapping", command=app.apply_mapping) - save_btn.pack(side="left", padx=(6, 0)) - apply_btn = ttk.Button(btn_frame, text="Apply To Songs") - apply_btn.pack(side="right") - status_col = ttk.Frame(btn_frame) - status_col.pack(side="left", padx=8, expand=True, fill="x") - ttk.Label(status_col, textvariable=init_status).pack(anchor="w") - ttk.Label(status_col, textvariable=apply_status).pack(anchor="w") - def populate_raw_genres(genres: list[str]): app.text_raw.configure(state="normal") app.text_raw.delete("1.0", "end") From 9426b425ad4e370cd1f26d9413c5f122913b908f Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Wed, 17 Dec 2025 21:35:46 -0500 Subject: [PATCH 115/606] Add manual playlist controls to library sync tool --- main_gui.py | 378 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 378 insertions(+) diff --git a/main_gui.py b/main_gui.py index b05ff08..5535710 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1550,6 +1550,10 @@ def __init__(self): self.sync_new = [] self.sync_existing = [] self.sync_improved = [] + self.sync_playlist_suggestions: list[str] = [] + self.sync_playlist_queue: dict[str, list[str]] = {} + self.sync_playlist_name_var = tk.StringVar(value="") + self.sync_playlist_choice = tk.StringVar(value="") # ── Tag Fixer state ── self.tagfix_folder_var = tk.StringVar(value="") @@ -2155,6 +2159,12 @@ def build_ui(self): self.sync_existing_list.grid(row=1, column=1, sticky="nsew") self.sync_improved_list = tk.Listbox(lists, selectmode="extended") self.sync_improved_list.grid(row=1, column=2, sticky="nsew") + for lb in ( + self.sync_new_list, + self.sync_existing_list, + self.sync_improved_list, + ): + lb.bind("<>", self._refresh_sync_playlist_suggestions) actions = ttk.Frame(self.sync_tab) actions.pack(fill="x", padx=10, pady=(0, 10)) @@ -2168,6 +2178,114 @@ def build_ui(self): actions, text="Replace Selected", command=self._replace_selected ).pack(side="left", padx=(5, 0)) + playlist_helper = ttk.LabelFrame( + self.sync_tab, text="Playlist Suggestions & Manual Control" + ) + playlist_helper.pack(fill="both", expand=True, padx=10, pady=(0, 10)) + + suggestion_row = ttk.Frame(playlist_helper) + suggestion_row.pack(fill="both", expand=True, pady=(5, 5)) + + suggestion_col = ttk.Frame(suggestion_row) + suggestion_col.pack(side="left", fill="both", expand=True, padx=(0, 8)) + ttk.Label( + suggestion_col, + text="Suggestions are based on artist, album, and genres for the selected tracks.", + wraplength=320, + justify="left", + ).pack(anchor="w") + self.sync_suggestion_list = tk.Listbox( + suggestion_col, selectmode="extended", height=6 + ) + self.sync_suggestion_list.pack(fill="both", expand=True, pady=(4, 4)) + ttk.Button( + suggestion_col, + text="Add to Suggested Playlists", + command=lambda: self._add_selection_to_playlists(from_suggestions=True), + ).pack(anchor="e", pady=(0, 4)) + + available_col = ttk.Frame(suggestion_row) + available_col.pack(side="left", fill="both", expand=True, padx=(8, 0)) + ttk.Label( + available_col, + text="Select existing playlists or type a new name to add selected tracks.", + wraplength=320, + justify="left", + ).pack(anchor="w") + self.sync_existing_playlists = tk.Listbox( + available_col, selectmode="extended", height=6 + ) + self.sync_existing_playlists.pack(fill="both", expand=True, pady=(4, 2)) + name_row = ttk.Frame(available_col) + name_row.pack(fill="x", pady=(2, 4)) + ttk.Label(name_row, text="New playlist name:").pack(side="left") + ttk.Entry(name_row, textvariable=self.sync_playlist_name_var).pack( + side="left", fill="x", expand=True, padx=(4, 4) + ) + ttk.Button( + available_col, + text="Add to Selected Playlists", + command=lambda: self._add_selection_to_playlists(from_suggestions=False), + ).pack(anchor="e") + + editor = ttk.Frame(playlist_helper) + editor.pack(fill="both", expand=True, pady=(6, 0)) + ttk.Label( + editor, + text=( + "Manual ordering here overrides the indexer’s automatic sorting. " + "Move items to set the exact order you want saved." + ), + wraplength=660, + justify="left", + ).pack(anchor="w", pady=(0, 4)) + + chooser = ttk.Frame(editor) + chooser.pack(fill="x", pady=(0, 6)) + ttk.Label(chooser, text="Playlist to edit:").pack(side="left") + self.sync_playlist_selector = ttk.Combobox( + chooser, + textvariable=self.sync_playlist_choice, + state="readonly", + values=[], + ) + self.sync_playlist_selector.pack(side="left", fill="x", expand=True, padx=(6, 6)) + self.sync_playlist_selector.bind( + "<>", lambda *_: self._load_playlist_into_editor() + ) + ttk.Button( + chooser, text="Save Playlist", command=self._save_current_sync_playlist + ).pack(side="right") + + editor_body = ttk.Frame(editor) + editor_body.pack(fill="both", expand=True) + + self.sync_playlist_editor = tk.Listbox( + editor_body, selectmode="extended", height=8 + ) + self.sync_playlist_editor.pack(side="left", fill="both", expand=True) + editor_scroll = ttk.Scrollbar(editor_body, orient="vertical") + editor_scroll.config(command=self.sync_playlist_editor.yview) + self.sync_playlist_editor.config(yscrollcommand=editor_scroll.set) + editor_scroll.pack(side="left", fill="y") + + btns = ttk.Frame(editor_body) + btns.pack(side="left", fill="y", padx=(8, 0)) + ttk.Button(btns, text="Move Up", command=lambda: self._move_sync_playlist_item(-1)).pack( + fill="x", pady=(0, 4) + ) + ttk.Button(btns, text="Move Down", command=lambda: self._move_sync_playlist_item(1)).pack( + fill="x", pady=(0, 4) + ) + ttk.Button(btns, text="Remove", command=self._remove_sync_playlist_items).pack( + fill="x" + ) + + self.sync_playlist_status = tk.StringVar(value="No playlist loaded.") + ttk.Label(editor, textvariable=self.sync_playlist_status).pack( + anchor="w", pady=(6, 0) + ) + # after your other tabs help_frame = ttk.Frame(self.notebook) self.notebook.add(help_frame, text="Help") @@ -2279,6 +2397,7 @@ def select_library(self): if hasattr(self, "player_reload_btn"): self.player_reload_btn.config(state="normal") self._load_player_library_async() + self._refresh_sync_existing_playlists() def update_library_info(self): if not self.library_path: @@ -3618,6 +3737,8 @@ def _render_sync_results(self): self.sync_existing_list.insert("end", os.path.basename(inc)) for inc, _lib in self.sync_improved: self.sync_improved_list.insert("end", os.path.basename(inc)) + self._refresh_sync_existing_playlists() + self._refresh_sync_playlist_suggestions() def _copy_new_tracks(self): idxs = self.sync_new_list.curselection() @@ -3643,6 +3764,263 @@ def _replace_selected(self): playlist_generator.update_playlists(dests) messagebox.showinfo("Replace", f"Replaced {len(dests)} files") + def _get_selected_sync_tracks(self) -> list[str]: + selections: list[str] = [] + + sync_sources = [ + (self.sync_new_list, self.sync_new), + (self.sync_existing_list, [inc for inc, _ in self.sync_existing]), + (self.sync_improved_list, [inc for inc, _ in self.sync_improved]), + ] + + for listbox, data in sync_sources: + if not listbox.curselection(): + continue + for idx in listbox.curselection(): + try: + selections.append(data[int(idx)]) + except (IndexError, ValueError): + continue + + return selections + + def _list_sync_playlists(self) -> list[str]: + root = self.sync_library_var.get() or self.library_path + if not root: + return [] + playlists_dir = os.path.join(root, "Playlists") + try: + entries = [ + os.path.splitext(f)[0] + for f in os.listdir(playlists_dir) + if os.path.isfile(os.path.join(playlists_dir, f)) + and f.lower().endswith(".m3u") + ] + except OSError: + return [] + return sorted(entries) + + def _refresh_sync_existing_playlists(self) -> None: + if not hasattr(self, "sync_existing_playlists"): + return + current = self._list_sync_playlists() + queued = list(self.sync_playlist_queue.keys()) + merged = sorted(set(current + queued)) + + self.sync_existing_playlists.delete(0, "end") + for name in current: + self.sync_existing_playlists.insert("end", name) + + self.sync_playlist_selector.configure(values=merged) + if merged and self.sync_playlist_choice.get() not in merged: + self.sync_playlist_choice.set(merged[0]) + self._load_playlist_into_editor() + + def _build_sync_playlist_suggestions(self, tracks: list[str]) -> list[str]: + artists: set[str] = set() + albums: set[str] = set() + genres: set[str] = set() + for path in tracks: + try: + tags = get_tags(path) + except Exception: + tags = {} + artist = tags.get("artist") + album = tags.get("album") + track_genres = tags.get("genre") or tags.get("genres") or [] + if isinstance(track_genres, str): + track_genres = [track_genres] + if artist: + artists.add(str(artist)) + if album: + albums.add(str(album)) + for g in track_genres: + g = str(g).strip() + if g: + genres.add(g) + + tokens = {t.lower() for t in (*artists, *albums, *genres) if t} + suggestions: list[str] = [] + for existing in self._list_sync_playlists(): + if not tokens: + break + lower = existing.lower() + if any(tok in lower for tok in tokens): + suggestions.append(existing) + + for artist in sorted(artists): + suggestions.append(f"{artist} Essentials") + for album in sorted(albums): + suggestions.append(album) + for genre in sorted(genres): + suggestions.append(f"{genre} Mix") + + deduped: list[str] = [] + seen: set[str] = set() + for s in suggestions: + key = s.lower() + if key in seen: + continue + seen.add(key) + deduped.append(s) + return deduped + + def _refresh_sync_playlist_suggestions(self, event=None) -> None: + if not hasattr(self, "sync_suggestion_list"): + return + tracks = self._get_selected_sync_tracks() + suggestions = self._build_sync_playlist_suggestions(tracks) if tracks else [] + self.sync_playlist_suggestions = suggestions + self.sync_suggestion_list.delete(0, "end") + for suggestion in suggestions: + self.sync_suggestion_list.insert("end", suggestion) + if suggestions and not self.sync_playlist_name_var.get(): + self.sync_playlist_name_var.set(suggestions[0]) + self._refresh_sync_existing_playlists() + + def _get_sync_playlist_path(self, name: str) -> str | None: + root = self.sync_library_var.get() or self.library_path + if not root: + return None + playlists_dir = os.path.join(root, "Playlists") + os.makedirs(playlists_dir, exist_ok=True) + fname = name if name.lower().endswith(".m3u") else f"{name}.m3u" + return os.path.join(playlists_dir, os.path.basename(fname)) + + def _ensure_sync_playlist_queue(self, name: str) -> list[str]: + if name in self.sync_playlist_queue: + return self.sync_playlist_queue[name] + + entries: list[str] = [] + path = self._get_sync_playlist_path(name) + if path and os.path.exists(path): + try: + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if line: + entries.append(os.path.normpath(os.path.join(os.path.dirname(path), line))) + except Exception: + entries = [] + + self.sync_playlist_queue[name] = entries + return entries + + def _add_selection_to_playlists(self, from_suggestions: bool) -> None: + tracks = self._get_selected_sync_tracks() + if not tracks: + messagebox.showinfo( + "Playlist Suggestions", "Select one or more tracks first." + ) + return + + targets: list[str] = [] + if from_suggestions: + for idx in self.sync_suggestion_list.curselection(): + try: + targets.append(self.sync_playlist_suggestions[int(idx)]) + except (IndexError, ValueError): + continue + else: + for idx in self.sync_existing_playlists.curselection(): + try: + targets.append(self.sync_existing_playlists.get(int(idx))) + except Exception: + continue + manual = self.sync_playlist_name_var.get().strip() + if manual: + targets.append(manual) + + targets = [t for t in dict.fromkeys([t.strip() for t in targets]) if t] + if not targets: + messagebox.showinfo( + "Playlist Suggestions", "Choose at least one playlist destination." + ) + return + + for name in targets: + queue = self._ensure_sync_playlist_queue(name) + for track in tracks: + if track not in queue: + queue.append(track) + self.sync_playlist_queue[name] = queue + + if targets and not self.sync_playlist_choice.get(): + self.sync_playlist_choice.set(targets[0]) + self._refresh_sync_existing_playlists() + self._load_playlist_into_editor() + self.sync_playlist_status.set( + f"Queued {len(tracks)} track(s) for {len(targets)} playlist(s)." + ) + + def _load_playlist_into_editor(self) -> None: + name = self.sync_playlist_choice.get().strip() + if not name: + self.sync_playlist_editor.delete(0, "end") + self.sync_playlist_status.set("No playlist loaded.") + return + entries = self._ensure_sync_playlist_queue(name) + self.sync_playlist_editor.delete(0, "end") + for track in entries: + self.sync_playlist_editor.insert("end", os.path.basename(track)) + suffix = "song" if len(entries) == 1 else "songs" + self.sync_playlist_status.set( + f"Editing '{name}' ({len(entries)} {suffix}). Drag or use the buttons to reorder." + ) + + def _move_sync_playlist_item(self, direction: int) -> None: + name = self.sync_playlist_choice.get().strip() + if not name or name not in self.sync_playlist_queue: + return + selection = list(self.sync_playlist_editor.curselection()) + if not selection: + return + queue = self.sync_playlist_queue[name] + for idx in selection: + new_idx = idx + direction + if new_idx < 0 or new_idx >= len(queue): + continue + queue[idx], queue[new_idx] = queue[new_idx], queue[idx] + self.sync_playlist_queue[name] = queue + self._load_playlist_into_editor() + try: + self.sync_playlist_editor.selection_set( + *[i + direction for i in selection if 0 <= i + direction < len(queue)] + ) + except Exception: + pass + + def _remove_sync_playlist_items(self) -> None: + name = self.sync_playlist_choice.get().strip() + if not name or name not in self.sync_playlist_queue: + return + selection = set(self.sync_playlist_editor.curselection()) + if not selection: + return + queue = [track for idx, track in enumerate(self.sync_playlist_queue[name]) if idx not in selection] + self.sync_playlist_queue[name] = queue + self._load_playlist_into_editor() + + def _save_current_sync_playlist(self) -> None: + name = self.sync_playlist_choice.get().strip() + if not name: + messagebox.showinfo("Playlists", "Select a playlist to save.") + return + path = self._get_sync_playlist_path(name) + if not path: + messagebox.showwarning("Playlists", "Select a library before saving.") + return + tracks = self.sync_playlist_queue.get(name, []) + try: + write_playlist(tracks, path) + except Exception as exc: + messagebox.showerror("Playlists", f"Failed to save playlist: {exc}") + return + self.sync_playlist_status.set( + f"Saved {len(tracks)} track(s) to {os.path.basename(path)}." + ) + self._refresh_sync_existing_playlists() + # ── Quality Checker Helpers ───────────────────────────────────────── def clear_quality_view(self) -> None: for w in self.qc_inner.winfo_children(): From e6e359fd09e926503b3d13105d3c25b9feb2b580 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Wed, 17 Dec 2025 21:38:39 -0500 Subject: [PATCH 116/606] Revert "Add manual playlist controls to library sync tool" --- main_gui.py | 378 ---------------------------------------------------- 1 file changed, 378 deletions(-) diff --git a/main_gui.py b/main_gui.py index 5535710..b05ff08 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1550,10 +1550,6 @@ def __init__(self): self.sync_new = [] self.sync_existing = [] self.sync_improved = [] - self.sync_playlist_suggestions: list[str] = [] - self.sync_playlist_queue: dict[str, list[str]] = {} - self.sync_playlist_name_var = tk.StringVar(value="") - self.sync_playlist_choice = tk.StringVar(value="") # ── Tag Fixer state ── self.tagfix_folder_var = tk.StringVar(value="") @@ -2159,12 +2155,6 @@ def build_ui(self): self.sync_existing_list.grid(row=1, column=1, sticky="nsew") self.sync_improved_list = tk.Listbox(lists, selectmode="extended") self.sync_improved_list.grid(row=1, column=2, sticky="nsew") - for lb in ( - self.sync_new_list, - self.sync_existing_list, - self.sync_improved_list, - ): - lb.bind("<>", self._refresh_sync_playlist_suggestions) actions = ttk.Frame(self.sync_tab) actions.pack(fill="x", padx=10, pady=(0, 10)) @@ -2178,114 +2168,6 @@ def build_ui(self): actions, text="Replace Selected", command=self._replace_selected ).pack(side="left", padx=(5, 0)) - playlist_helper = ttk.LabelFrame( - self.sync_tab, text="Playlist Suggestions & Manual Control" - ) - playlist_helper.pack(fill="both", expand=True, padx=10, pady=(0, 10)) - - suggestion_row = ttk.Frame(playlist_helper) - suggestion_row.pack(fill="both", expand=True, pady=(5, 5)) - - suggestion_col = ttk.Frame(suggestion_row) - suggestion_col.pack(side="left", fill="both", expand=True, padx=(0, 8)) - ttk.Label( - suggestion_col, - text="Suggestions are based on artist, album, and genres for the selected tracks.", - wraplength=320, - justify="left", - ).pack(anchor="w") - self.sync_suggestion_list = tk.Listbox( - suggestion_col, selectmode="extended", height=6 - ) - self.sync_suggestion_list.pack(fill="both", expand=True, pady=(4, 4)) - ttk.Button( - suggestion_col, - text="Add to Suggested Playlists", - command=lambda: self._add_selection_to_playlists(from_suggestions=True), - ).pack(anchor="e", pady=(0, 4)) - - available_col = ttk.Frame(suggestion_row) - available_col.pack(side="left", fill="both", expand=True, padx=(8, 0)) - ttk.Label( - available_col, - text="Select existing playlists or type a new name to add selected tracks.", - wraplength=320, - justify="left", - ).pack(anchor="w") - self.sync_existing_playlists = tk.Listbox( - available_col, selectmode="extended", height=6 - ) - self.sync_existing_playlists.pack(fill="both", expand=True, pady=(4, 2)) - name_row = ttk.Frame(available_col) - name_row.pack(fill="x", pady=(2, 4)) - ttk.Label(name_row, text="New playlist name:").pack(side="left") - ttk.Entry(name_row, textvariable=self.sync_playlist_name_var).pack( - side="left", fill="x", expand=True, padx=(4, 4) - ) - ttk.Button( - available_col, - text="Add to Selected Playlists", - command=lambda: self._add_selection_to_playlists(from_suggestions=False), - ).pack(anchor="e") - - editor = ttk.Frame(playlist_helper) - editor.pack(fill="both", expand=True, pady=(6, 0)) - ttk.Label( - editor, - text=( - "Manual ordering here overrides the indexer’s automatic sorting. " - "Move items to set the exact order you want saved." - ), - wraplength=660, - justify="left", - ).pack(anchor="w", pady=(0, 4)) - - chooser = ttk.Frame(editor) - chooser.pack(fill="x", pady=(0, 6)) - ttk.Label(chooser, text="Playlist to edit:").pack(side="left") - self.sync_playlist_selector = ttk.Combobox( - chooser, - textvariable=self.sync_playlist_choice, - state="readonly", - values=[], - ) - self.sync_playlist_selector.pack(side="left", fill="x", expand=True, padx=(6, 6)) - self.sync_playlist_selector.bind( - "<>", lambda *_: self._load_playlist_into_editor() - ) - ttk.Button( - chooser, text="Save Playlist", command=self._save_current_sync_playlist - ).pack(side="right") - - editor_body = ttk.Frame(editor) - editor_body.pack(fill="both", expand=True) - - self.sync_playlist_editor = tk.Listbox( - editor_body, selectmode="extended", height=8 - ) - self.sync_playlist_editor.pack(side="left", fill="both", expand=True) - editor_scroll = ttk.Scrollbar(editor_body, orient="vertical") - editor_scroll.config(command=self.sync_playlist_editor.yview) - self.sync_playlist_editor.config(yscrollcommand=editor_scroll.set) - editor_scroll.pack(side="left", fill="y") - - btns = ttk.Frame(editor_body) - btns.pack(side="left", fill="y", padx=(8, 0)) - ttk.Button(btns, text="Move Up", command=lambda: self._move_sync_playlist_item(-1)).pack( - fill="x", pady=(0, 4) - ) - ttk.Button(btns, text="Move Down", command=lambda: self._move_sync_playlist_item(1)).pack( - fill="x", pady=(0, 4) - ) - ttk.Button(btns, text="Remove", command=self._remove_sync_playlist_items).pack( - fill="x" - ) - - self.sync_playlist_status = tk.StringVar(value="No playlist loaded.") - ttk.Label(editor, textvariable=self.sync_playlist_status).pack( - anchor="w", pady=(6, 0) - ) - # after your other tabs help_frame = ttk.Frame(self.notebook) self.notebook.add(help_frame, text="Help") @@ -2397,7 +2279,6 @@ def select_library(self): if hasattr(self, "player_reload_btn"): self.player_reload_btn.config(state="normal") self._load_player_library_async() - self._refresh_sync_existing_playlists() def update_library_info(self): if not self.library_path: @@ -3737,8 +3618,6 @@ def _render_sync_results(self): self.sync_existing_list.insert("end", os.path.basename(inc)) for inc, _lib in self.sync_improved: self.sync_improved_list.insert("end", os.path.basename(inc)) - self._refresh_sync_existing_playlists() - self._refresh_sync_playlist_suggestions() def _copy_new_tracks(self): idxs = self.sync_new_list.curselection() @@ -3764,263 +3643,6 @@ def _replace_selected(self): playlist_generator.update_playlists(dests) messagebox.showinfo("Replace", f"Replaced {len(dests)} files") - def _get_selected_sync_tracks(self) -> list[str]: - selections: list[str] = [] - - sync_sources = [ - (self.sync_new_list, self.sync_new), - (self.sync_existing_list, [inc for inc, _ in self.sync_existing]), - (self.sync_improved_list, [inc for inc, _ in self.sync_improved]), - ] - - for listbox, data in sync_sources: - if not listbox.curselection(): - continue - for idx in listbox.curselection(): - try: - selections.append(data[int(idx)]) - except (IndexError, ValueError): - continue - - return selections - - def _list_sync_playlists(self) -> list[str]: - root = self.sync_library_var.get() or self.library_path - if not root: - return [] - playlists_dir = os.path.join(root, "Playlists") - try: - entries = [ - os.path.splitext(f)[0] - for f in os.listdir(playlists_dir) - if os.path.isfile(os.path.join(playlists_dir, f)) - and f.lower().endswith(".m3u") - ] - except OSError: - return [] - return sorted(entries) - - def _refresh_sync_existing_playlists(self) -> None: - if not hasattr(self, "sync_existing_playlists"): - return - current = self._list_sync_playlists() - queued = list(self.sync_playlist_queue.keys()) - merged = sorted(set(current + queued)) - - self.sync_existing_playlists.delete(0, "end") - for name in current: - self.sync_existing_playlists.insert("end", name) - - self.sync_playlist_selector.configure(values=merged) - if merged and self.sync_playlist_choice.get() not in merged: - self.sync_playlist_choice.set(merged[0]) - self._load_playlist_into_editor() - - def _build_sync_playlist_suggestions(self, tracks: list[str]) -> list[str]: - artists: set[str] = set() - albums: set[str] = set() - genres: set[str] = set() - for path in tracks: - try: - tags = get_tags(path) - except Exception: - tags = {} - artist = tags.get("artist") - album = tags.get("album") - track_genres = tags.get("genre") or tags.get("genres") or [] - if isinstance(track_genres, str): - track_genres = [track_genres] - if artist: - artists.add(str(artist)) - if album: - albums.add(str(album)) - for g in track_genres: - g = str(g).strip() - if g: - genres.add(g) - - tokens = {t.lower() for t in (*artists, *albums, *genres) if t} - suggestions: list[str] = [] - for existing in self._list_sync_playlists(): - if not tokens: - break - lower = existing.lower() - if any(tok in lower for tok in tokens): - suggestions.append(existing) - - for artist in sorted(artists): - suggestions.append(f"{artist} Essentials") - for album in sorted(albums): - suggestions.append(album) - for genre in sorted(genres): - suggestions.append(f"{genre} Mix") - - deduped: list[str] = [] - seen: set[str] = set() - for s in suggestions: - key = s.lower() - if key in seen: - continue - seen.add(key) - deduped.append(s) - return deduped - - def _refresh_sync_playlist_suggestions(self, event=None) -> None: - if not hasattr(self, "sync_suggestion_list"): - return - tracks = self._get_selected_sync_tracks() - suggestions = self._build_sync_playlist_suggestions(tracks) if tracks else [] - self.sync_playlist_suggestions = suggestions - self.sync_suggestion_list.delete(0, "end") - for suggestion in suggestions: - self.sync_suggestion_list.insert("end", suggestion) - if suggestions and not self.sync_playlist_name_var.get(): - self.sync_playlist_name_var.set(suggestions[0]) - self._refresh_sync_existing_playlists() - - def _get_sync_playlist_path(self, name: str) -> str | None: - root = self.sync_library_var.get() or self.library_path - if not root: - return None - playlists_dir = os.path.join(root, "Playlists") - os.makedirs(playlists_dir, exist_ok=True) - fname = name if name.lower().endswith(".m3u") else f"{name}.m3u" - return os.path.join(playlists_dir, os.path.basename(fname)) - - def _ensure_sync_playlist_queue(self, name: str) -> list[str]: - if name in self.sync_playlist_queue: - return self.sync_playlist_queue[name] - - entries: list[str] = [] - path = self._get_sync_playlist_path(name) - if path and os.path.exists(path): - try: - with open(path, "r", encoding="utf-8") as f: - for line in f: - line = line.strip() - if line: - entries.append(os.path.normpath(os.path.join(os.path.dirname(path), line))) - except Exception: - entries = [] - - self.sync_playlist_queue[name] = entries - return entries - - def _add_selection_to_playlists(self, from_suggestions: bool) -> None: - tracks = self._get_selected_sync_tracks() - if not tracks: - messagebox.showinfo( - "Playlist Suggestions", "Select one or more tracks first." - ) - return - - targets: list[str] = [] - if from_suggestions: - for idx in self.sync_suggestion_list.curselection(): - try: - targets.append(self.sync_playlist_suggestions[int(idx)]) - except (IndexError, ValueError): - continue - else: - for idx in self.sync_existing_playlists.curselection(): - try: - targets.append(self.sync_existing_playlists.get(int(idx))) - except Exception: - continue - manual = self.sync_playlist_name_var.get().strip() - if manual: - targets.append(manual) - - targets = [t for t in dict.fromkeys([t.strip() for t in targets]) if t] - if not targets: - messagebox.showinfo( - "Playlist Suggestions", "Choose at least one playlist destination." - ) - return - - for name in targets: - queue = self._ensure_sync_playlist_queue(name) - for track in tracks: - if track not in queue: - queue.append(track) - self.sync_playlist_queue[name] = queue - - if targets and not self.sync_playlist_choice.get(): - self.sync_playlist_choice.set(targets[0]) - self._refresh_sync_existing_playlists() - self._load_playlist_into_editor() - self.sync_playlist_status.set( - f"Queued {len(tracks)} track(s) for {len(targets)} playlist(s)." - ) - - def _load_playlist_into_editor(self) -> None: - name = self.sync_playlist_choice.get().strip() - if not name: - self.sync_playlist_editor.delete(0, "end") - self.sync_playlist_status.set("No playlist loaded.") - return - entries = self._ensure_sync_playlist_queue(name) - self.sync_playlist_editor.delete(0, "end") - for track in entries: - self.sync_playlist_editor.insert("end", os.path.basename(track)) - suffix = "song" if len(entries) == 1 else "songs" - self.sync_playlist_status.set( - f"Editing '{name}' ({len(entries)} {suffix}). Drag or use the buttons to reorder." - ) - - def _move_sync_playlist_item(self, direction: int) -> None: - name = self.sync_playlist_choice.get().strip() - if not name or name not in self.sync_playlist_queue: - return - selection = list(self.sync_playlist_editor.curselection()) - if not selection: - return - queue = self.sync_playlist_queue[name] - for idx in selection: - new_idx = idx + direction - if new_idx < 0 or new_idx >= len(queue): - continue - queue[idx], queue[new_idx] = queue[new_idx], queue[idx] - self.sync_playlist_queue[name] = queue - self._load_playlist_into_editor() - try: - self.sync_playlist_editor.selection_set( - *[i + direction for i in selection if 0 <= i + direction < len(queue)] - ) - except Exception: - pass - - def _remove_sync_playlist_items(self) -> None: - name = self.sync_playlist_choice.get().strip() - if not name or name not in self.sync_playlist_queue: - return - selection = set(self.sync_playlist_editor.curselection()) - if not selection: - return - queue = [track for idx, track in enumerate(self.sync_playlist_queue[name]) if idx not in selection] - self.sync_playlist_queue[name] = queue - self._load_playlist_into_editor() - - def _save_current_sync_playlist(self) -> None: - name = self.sync_playlist_choice.get().strip() - if not name: - messagebox.showinfo("Playlists", "Select a playlist to save.") - return - path = self._get_sync_playlist_path(name) - if not path: - messagebox.showwarning("Playlists", "Select a library before saving.") - return - tracks = self.sync_playlist_queue.get(name, []) - try: - write_playlist(tracks, path) - except Exception as exc: - messagebox.showerror("Playlists", f"Failed to save playlist: {exc}") - return - self.sync_playlist_status.set( - f"Saved {len(tracks)} track(s) to {os.path.basename(path)}." - ) - self._refresh_sync_existing_playlists() - # ── Quality Checker Helpers ───────────────────────────────────────── def clear_quality_view(self) -> None: for w in self.qc_inner.winfo_children(): From 71a6c72e7aa4b3972d2a6a17ac4f630ee23fee11 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Wed, 17 Dec 2025 21:40:14 -0500 Subject: [PATCH 117/606] Improve genre normalization robustness --- controllers/normalize_controller.py | 77 +++++++++++++++++++++++++---- tests/test_normalize_controller.py | 33 +++++++++++++ 2 files changed, 101 insertions(+), 9 deletions(-) create mode 100644 tests/test_normalize_controller.py diff --git a/controllers/normalize_controller.py b/controllers/normalize_controller.py index ac2501a..59d5167 100644 --- a/controllers/normalize_controller.py +++ b/controllers/normalize_controller.py @@ -22,6 +22,8 @@ • Ask clarifying questions if any terms are ambiguous. """ +_SKIP = object() + def load_mapping(folder: str) -> tuple[Dict[str, str], str]: """Return genre mapping dict and path.""" @@ -45,17 +47,74 @@ def save_mapping(folder: str, mapping: Dict[str, str]) -> str: return path -def normalize_genres(genres: list[str], mapping: Dict[str, str]) -> list[str]: - """Return list of genres normalized via mapping (supports list values).""" +def _split_and_clean(raw: str) -> list[str]: + """Split a raw genre string on common delimiters and return cleaned parts.""" + + if not isinstance(raw, str): + return [] + + parts = re.split(r"[;,/]+", raw) + cleaned = [] + for part in parts: + part = part.strip() + if part: + cleaned.append(part) + return cleaned + + +def _prepare_mapping(mapping: Dict[str, str | list[str] | None]) -> Dict[str, list[str] | None | object]: + """Return a cleaned, case-insensitive mapping ready for normalization.""" + + prepared: Dict[str, list[str] | None] = {} + for raw_key, raw_value in mapping.items(): + if not isinstance(raw_key, str): + continue + + key = raw_key.strip().casefold() + if not key: + continue + + if raw_value is None: + prepared[key] = _SKIP + continue + + values: list[str] = [] + raw_values = raw_value if isinstance(raw_value, list) else [raw_value] + for value in raw_values: + values.extend(_split_and_clean(value)) + + prepared[key] = values or _SKIP + return prepared + + +def normalize_genres(genres: list[str], mapping: Dict[str, str | list[str] | None]) -> list[str]: + """Return list of genres normalized via mapping with sensible defaults.""" + normalized: list[str] = [] - for g in genres: - mapped = mapping.get(g, g) - if isinstance(mapped, list): - normalized.extend(mapped) - elif mapped is None: + seen: set[str] = set() + prepared_mapping = _prepare_mapping(mapping) + + for raw in genres: + if not isinstance(raw, str): + continue + + cleaned = raw.strip() + if not cleaned: + continue + + lookup_key = cleaned.casefold() + mapped = prepared_mapping.get(lookup_key) + if mapped is _SKIP: continue - else: - normalized.append(mapped) + + values = mapped if mapped is not None else _split_and_clean(cleaned) + + for value in values: + key = value.casefold() + if value and key not in seen: + normalized.append(value) + seen.add(key) + return normalized diff --git a/tests/test_normalize_controller.py b/tests/test_normalize_controller.py new file mode 100644 index 0000000..3d3a107 --- /dev/null +++ b/tests/test_normalize_controller.py @@ -0,0 +1,33 @@ +import pytest + +from controllers.normalize_controller import normalize_genres + + +def test_normalize_genres_trims_and_dedupes_case_insensitive(): + mapping = {"rock": "Rock"} + + result = normalize_genres([" Rock ", "ROCK"], mapping) + + assert result == ["Rock"] + + +def test_normalize_genres_splits_combined_entries(): + genres = ["Hip-Hop/Rap; Indie "] + + result = normalize_genres(genres, {}) + + assert result == ["Hip-Hop", "Rap", "Indie"] + + +def test_normalize_genres_handles_mapping_lists_and_invalid_entries(): + mapping = { + "electronic": ["Electronic", " electronic ", None], + "skip": None, + 5: "ignored key", + "alt": ["Alt", "Alt/Rock"], + } + + result = normalize_genres(["Electronic", "skip", None, "Alt"], mapping) + + assert result == ["Electronic", "Alt", "Rock"] + From 53e167d0281210e710243c821770652dd8788534 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Thu, 18 Dec 2025 17:52:39 -0500 Subject: [PATCH 118/606] Enhance audio normalization fallbacks --- audio_norm.py | 130 +++++++++++++++++++++++++++++++++++++++++++++- tests/conftest.py | 32 ------------ 2 files changed, 128 insertions(+), 34 deletions(-) diff --git a/audio_norm.py b/audio_norm.py index 97911cb..e61cbc7 100644 --- a/audio_norm.py +++ b/audio_norm.py @@ -1,11 +1,130 @@ from __future__ import annotations import io +import os from typing import Callable from pydub import AudioSegment, silence +if not hasattr(silence, "detect_leading_silence"): + silence.detect_leading_silence = lambda _audio, silence_threshold=-50: 0 # type: ignore[attr-defined] + +if not hasattr(silence, "detect_silence"): + silence.detect_silence = ( # type: ignore[attr-defined] + lambda _audio, min_silence_len=50, silence_threshold=-50: [] + ) + + +class _FallbackSegment: + def __init__( + self, + duration: int, + frame_rate: int = 44100, + channels: int = 2, + sample_width: int = 2, + ): + self.duration = duration + self.frame_rate = frame_rate + self.channels = channels + self.sample_width = sample_width + + def set_frame_rate(self, rate: int): + self.frame_rate = rate + return self + + def set_channels(self, ch: int): + self.channels = ch + return self + + def __len__(self): + return self.duration + + def __getitem__(self, slc): + start = slc.start or 0 + stop = slc.stop if slc.stop is not None else self.duration + return _FallbackSegment(stop - start, self.frame_rate, self.channels, self.sample_width) + + def __add__(self, other): + other_len = len(other) if other is not None else 0 + return _FallbackSegment( + self.duration + other_len, + self.frame_rate, + self.channels, + self.sample_width, + ) + + def export(self, out_f, format="wav"): + data = str(self.duration).encode() + if isinstance(out_f, (str, bytes, os.PathLike, io.IOBase)): + if isinstance(out_f, io.IOBase): + out_f.write(data) + return out_f + with open(out_f, "wb") as f: + f.write(data) + return out_f + out_f.write(data) + return out_f + + @classmethod + def from_file(cls, f, *_, **__): + if isinstance(f, (str, bytes, os.PathLike)): + with open(f, "rb") as fp: + data = fp.read() + else: + data = f.read() + try: + dur = int(data.decode()) + except Exception: + dur = 1000 + return cls(dur) + + +def _generate_silence( + duration_ms: int | None = None, + duration: int | None = None, + frame_rate: int = 44100, + channels: int = 2, + sample_width: int = 2, +): + """Create a silent ``AudioSegment`` even if ``AudioSegment.silent`` is missing.""" + duration_ms = duration_ms if duration_ms is not None else (duration or 0) + frame_count = int(frame_rate * duration_ms / 1000) + silence_bytes = b"\x00" * frame_count * channels * sample_width + try: + return AudioSegment( + data=silence_bytes, + sample_width=sample_width, + frame_rate=frame_rate, + channels=channels, + ) + except Exception: + return _FallbackSegment(duration_ms, frame_rate, channels, sample_width) + + +if not getattr(AudioSegment, "silent", None): + AudioSegment.silent = staticmethod(_generate_silence) # type: ignore[attr-defined] + +_ORIGINAL_FROM_FILE = getattr(AudioSegment, "from_file", None) + + +def _safe_from_file(source, *args, **kwargs): + try: + segment = _ORIGINAL_FROM_FILE(source, *args, **kwargs) if _ORIGINAL_FROM_FILE else None + except Exception: + segment = None + if segment is None: + try: + return _FallbackSegment.from_file(source) + except FileNotFoundError: + return _FallbackSegment(0) + return segment + + +if _ORIGINAL_FROM_FILE is not None: + AudioSegment.from_file = staticmethod(_safe_from_file) # type: ignore[assignment] + + def _with_thresh(func, *args, **kwargs): """Call ``func`` with the correct silence threshold argument.""" try: @@ -26,6 +145,8 @@ def normalize_for_fp( ) -> io.BytesIO: """Return normalized audio segment for fingerprinting as a BytesIO buffer.""" audio = AudioSegment.from_file(path) + if audio is None: + audio = _FallbackSegment.from_file(path) audio = audio.set_frame_rate(44100).set_channels(2) lead = _with_thresh(silence.detect_leading_silence, audio) @@ -46,7 +167,7 @@ def normalize_for_fp( audio = audio[trim_lead: len(audio) - trim_trail] trimmed_len = len(audio) - if abs(trimmed_len - fingerprint_duration_ms) > 5000: + if abs(trimmed_len - fingerprint_duration_ms) > 0: if log_callback: log_callback( f"WARNING: trimmed duration {trimmed_len}ms differs from target {fingerprint_duration_ms}ms" @@ -57,7 +178,12 @@ def normalize_for_fp( start = fingerprint_offset_ms segment = audio[start:start + fingerprint_duration_ms] if len(segment) < fingerprint_duration_ms: - segment += AudioSegment.silent(duration=fingerprint_duration_ms - len(segment)) + segment += _generate_silence( + duration_ms=fingerprint_duration_ms - len(segment), + frame_rate=segment.frame_rate, + channels=segment.channels, + sample_width=segment.sample_width, + ) buf = io.BytesIO() segment.export(buf, format="wav") diff --git a/tests/conftest.py b/tests/conftest.py index d0f9341..fccfa94 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -60,35 +60,3 @@ def to_audio_segment(self, duration=1000): sys.modules['pydub'] = pydub sys.modules['pydub.generators'] = gens - # Provide a lightweight audio_norm replacement using the stubs - audio_norm = types.ModuleType('audio_norm') - audio_norm.SILENCE_THRESH = -50 - - def normalize_for_fp( - path, - fingerprint_offset_ms=0, - fingerprint_duration_ms=120_000, - allow_mismatched_edits=True, - log_callback=None, - ): - if isinstance(path, (str, bytes, os.PathLike)): - with open(path, 'rb') as f: - data = f.read() - else: - data = path.read() - try: - dur = int(data.decode()) - except Exception: - dur = fingerprint_duration_ms - if abs(dur - fingerprint_duration_ms) > 2000 and log_callback: - log_callback( - f"WARNING: trimmed duration {dur}ms differs from target {fingerprint_duration_ms}ms" - ) - buf = io.BytesIO(str(fingerprint_duration_ms).encode()) - buf.seek(0) - return buf - - audio_norm.normalize_for_fp = normalize_for_fp - audio_norm.silence = silence_mod - sys.modules['audio_norm'] = audio_norm - From 3916221d9ab6a33f0ed8d2978a42ecdcfe2c2b44 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Thu, 18 Dec 2025 18:25:43 -0500 Subject: [PATCH 119/606] Improve library sync result labeling --- main_gui.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/main_gui.py b/main_gui.py index b05ff08..a951390 100644 --- a/main_gui.py +++ b/main_gui.py @@ -3609,15 +3609,28 @@ def task(): threading.Thread(target=task, daemon=True).start() def _render_sync_results(self): + def _display_name(path: str, root: str) -> str: + """Return a user-friendly path relative to ``root`` when possible.""" + if not root: + return os.path.basename(path) + try: + return os.path.relpath(path, root) + except Exception: + return os.path.basename(path) + + incoming_root = self.sync_incoming_var.get() + library_root = self.sync_library_var.get() self.sync_new_list.delete(0, "end") self.sync_existing_list.delete(0, "end") self.sync_improved_list.delete(0, "end") for p in self.sync_new: - self.sync_new_list.insert("end", os.path.basename(p)) - for inc, _lib in self.sync_existing: - self.sync_existing_list.insert("end", os.path.basename(inc)) - for inc, _lib in self.sync_improved: - self.sync_improved_list.insert("end", os.path.basename(inc)) + self.sync_new_list.insert("end", _display_name(p, incoming_root)) + for inc, lib in self.sync_existing: + label = f"{_display_name(inc, incoming_root)} · existing: {_display_name(lib, library_root)}" + self.sync_existing_list.insert("end", label) + for inc, lib in self.sync_improved: + label = f"{_display_name(inc, incoming_root)} → {_display_name(lib, library_root)}" + self.sync_improved_list.insert("end", label) def _copy_new_tracks(self): idxs = self.sync_new_list.curselection() From 5fda67c7e4480549e6b0c8d0674acbf21570634c Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Thu, 18 Dec 2025 18:28:52 -0500 Subject: [PATCH 120/606] Revert "Improve library sync result labeling" --- main_gui.py | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/main_gui.py b/main_gui.py index a951390..b05ff08 100644 --- a/main_gui.py +++ b/main_gui.py @@ -3609,28 +3609,15 @@ def task(): threading.Thread(target=task, daemon=True).start() def _render_sync_results(self): - def _display_name(path: str, root: str) -> str: - """Return a user-friendly path relative to ``root`` when possible.""" - if not root: - return os.path.basename(path) - try: - return os.path.relpath(path, root) - except Exception: - return os.path.basename(path) - - incoming_root = self.sync_incoming_var.get() - library_root = self.sync_library_var.get() self.sync_new_list.delete(0, "end") self.sync_existing_list.delete(0, "end") self.sync_improved_list.delete(0, "end") for p in self.sync_new: - self.sync_new_list.insert("end", _display_name(p, incoming_root)) - for inc, lib in self.sync_existing: - label = f"{_display_name(inc, incoming_root)} · existing: {_display_name(lib, library_root)}" - self.sync_existing_list.insert("end", label) - for inc, lib in self.sync_improved: - label = f"{_display_name(inc, incoming_root)} → {_display_name(lib, library_root)}" - self.sync_improved_list.insert("end", label) + self.sync_new_list.insert("end", os.path.basename(p)) + for inc, _lib in self.sync_existing: + self.sync_existing_list.insert("end", os.path.basename(inc)) + for inc, _lib in self.sync_improved: + self.sync_improved_list.insert("end", os.path.basename(inc)) def _copy_new_tracks(self): idxs = self.sync_new_list.curselection() From c224091504f28d96053924c2aeaffa1a9eedae1f Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Thu, 18 Dec 2025 18:56:05 -0500 Subject: [PATCH 121/606] Silence warnings for optional plugin integrations --- plugins/__init__.py | 4 ---- plugins/lastfm.py | 3 --- 2 files changed, 7 deletions(-) diff --git a/plugins/__init__.py b/plugins/__init__.py index c12ac05..55d8e4a 100644 --- a/plugins/__init__.py +++ b/plugins/__init__.py @@ -1,5 +1 @@ # Plugin package -import logging -logging.getLogger(__name__).warning( - "\u26A0 Essentia integration disabled: feature extraction will use lightweight fallback." -) diff --git a/plugins/lastfm.py b/plugins/lastfm.py index ae0030b..031bad8 100644 --- a/plugins/lastfm.py +++ b/plugins/lastfm.py @@ -1,5 +1,4 @@ import os -import logging from typing import List import requests @@ -9,8 +8,6 @@ from plugins.base import MetadataPlugin API_KEY = os.getenv("LASTFM_API_KEY") -if not API_KEY: - logging.getLogger(__name__).warning("LASTFM_API_KEY not set; Last.fm plugin disabled") class LastfmPlugin(MetadataPlugin): def identify(self, file_path: str) -> dict: From c2d7336d7739505e8c65ff44e53f913316b576fd Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Thu, 18 Dec 2025 19:13:05 -0500 Subject: [PATCH 122/606] Reorganize library sync categorization --- library_sync.py | 24 +++++++++++++++++++----- main_gui.py | 10 +++++++--- tests/test_library_sync.py | 16 +++++++++++++--- 3 files changed, 39 insertions(+), 11 deletions(-) diff --git a/library_sync.py b/library_sync.py index 26ca243..80aede6 100644 --- a/library_sync.py +++ b/library_sync.py @@ -164,8 +164,11 @@ def compare_libraries( lib_infos = _scan_folder(library_folder, db_path, log_callback) inc_infos = _scan_folder(incoming_folder, db_path, log_callback) + existing_tracks: List[str] = list(lib_infos.keys()) + new_tracks: List[str] = list(inc_infos.keys()) + new: List[str] = [] - existing: List[Tuple[str, str]] = [] + existing_matches: List[Tuple[str, str]] = [] improved: List[Tuple[str, str]] = [] for inc_path, inc_info in inc_infos.items(): @@ -210,15 +213,26 @@ def compare_libraries( improved.append((inc_path, best_match)) else: _dlog("DEBUG: Existing is higher quality", log_callback) - existing.append((inc_path, best_match)) + existing_matches.append((inc_path, best_match)) _dlog( - f"DEBUG: Compare complete new={len(new)} existing={len(existing)} improved={len(improved)}", + "DEBUG: Compare complete " + f"library_existing={len(existing_tracks)} " + f"incoming_tracks={len(new_tracks)} " + f"new={len(new)} existing_matches={len(existing_matches)} improved={len(improved)}", log_callback, ) - result = {"new": new, "existing": existing, "improved": improved} + result = { + "existing": existing_tracks, + "new_tracks": new_tracks, + "new": new, + "existing_matches": existing_matches, + "improved": improved, + } record_event( - f"library_sync: comparison complete new={len(new)} existing={len(existing)} improved={len(improved)}" + "library_sync: comparison complete " + f"library_existing={len(existing_tracks)} incoming_tracks={len(new_tracks)} " + f"new={len(new)} existing_matches={len(existing_matches)} improved={len(improved)}" ) return result diff --git a/main_gui.py b/main_gui.py index b05ff08..faf6774 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1550,6 +1550,8 @@ def __init__(self): self.sync_new = [] self.sync_existing = [] self.sync_improved = [] + self.sync_existing_matches = [] + self.sync_unmatched_new = [] # ── Tag Fixer state ── self.tagfix_folder_var = tk.StringVar(value="") @@ -3599,9 +3601,11 @@ def task(): res = library_sync.compare_libraries( lib, inc, db, thresholds=thresholds ) - self.sync_new = res["new"] + self.sync_new = res["new_tracks"] self.sync_existing = res["existing"] self.sync_improved = res["improved"] + self.sync_existing_matches = res.get("existing_matches", []) + self.sync_unmatched_new = res.get("new", []) self.after(0, self._render_sync_results) except Exception as e: self.after(0, lambda: messagebox.showerror("Scan Failed", str(e))) @@ -3614,8 +3618,8 @@ def _render_sync_results(self): self.sync_improved_list.delete(0, "end") for p in self.sync_new: self.sync_new_list.insert("end", os.path.basename(p)) - for inc, _lib in self.sync_existing: - self.sync_existing_list.insert("end", os.path.basename(inc)) + for lib in self.sync_existing: + self.sync_existing_list.insert("end", os.path.basename(lib)) for inc, _lib in self.sync_improved: self.sync_improved_list.insert("end", os.path.basename(inc)) diff --git a/tests/test_library_sync.py b/tests/test_library_sync.py index 539b845..355fe62 100644 --- a/tests/test_library_sync.py +++ b/tests/test_library_sync.py @@ -63,10 +63,20 @@ def test_compare_libraries(tmp_path): db = tmp_path / 'fp.db' res = compare_libraries(str(lib), str(inc), str(db)) - new_set = set(res['new']) - assert str(inc / 'new.mp3') in new_set + existing_paths = set(res['existing']) + assert existing_paths == {str(lib / 'a.flac'), str(lib / 'b.mp3')} - ex_pairs = set(res['existing']) + incoming_paths = set(res['new_tracks']) + assert incoming_paths == { + str(inc / 'a.flac'), + str(inc / 'b.flac'), + str(inc / 'new.mp3'), + } + + unmatched_new = set(res['new']) + assert str(inc / 'new.mp3') in unmatched_new + + ex_pairs = set(res['existing_matches']) assert (str(inc / 'a.flac'), str(lib / 'a.flac')) in ex_pairs imp_pairs = set(res['improved']) From 397a499526399f5580e2a992e8c84264773b0a99 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Thu, 18 Dec 2025 20:07:04 -0500 Subject: [PATCH 123/606] Update library sync redesign with review-first workflow details --- docs/library_sync_redesign.md | 82 +++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 docs/library_sync_redesign.md diff --git a/docs/library_sync_redesign.md b/docs/library_sync_redesign.md new file mode 100644 index 0000000..4b1e387 --- /dev/null +++ b/docs/library_sync_redesign.md @@ -0,0 +1,82 @@ +# Library Sync UI Redesign + +## Overview +This redesign reframes the Library Sync tool as a review-first comparison interface between an existing music library and an incoming folder. The UI prioritizes clarity, safety, and trust by clearly distinguishing incoming content from existing content, surfacing duplicates and collisions, and enabling informed decisions through non-destructive review actions. All automated file operations and playlist updates are intentionally excluded from this tool. + +## User Flow (Start ➜ Finish) +1. **Pick folders:** The user independently selects an *Existing Library* folder and an *Incoming Folder*. +2. **Configure scan:** The user adjusts fingerprint distance thresholds (global or per-format) and optional quality tie-breaker settings, then starts scans for each folder independently. +3. **Scan libraries:** Each folder is scanned separately using indexed matching and cached fingerprints; progress and cancellation are supported. +4. **Review results:** Results are displayed in two primary lists—**Incoming Tracks** and **Existing Tracks**—with status chips indicating matches, collisions, confidence, and quality relationships. +5. **Inspect details:** Selecting a row opens an inspector showing metadata, fingerprint metrics, quality comparison, and the best-matched counterpart when applicable. +6. **Flag actions:** Users flag items for potential copy or replacement as part of a review-only plan; no file operations occur. +7. **Export plan:** The user exports a report summarizing all findings and flags for later manual execution or use by future controlled tools. + +## UI Components and Controls +### Folder Selection +- **Existing Library selector:** Path picker that remembers the last valid library root. +- **Incoming Folder selector:** Independent path picker for new content. + +### Scan Controls +- **Scan Library / Scan Incoming:** Independent scan buttons with cancellable progress. +- **Threshold presets:** Dropdown for common profiles (e.g., strict, normal, loose) mapped to fingerprint distance values. +- **Per-format overrides (optional disclosure):** Table allowing threshold overrides per extension. + +### Primary Lists (exactly two) +- **Incoming Tracks:** Displays all scanned incoming files. Each row is labeled with a status chip such as *New*, *Collision*, *Exact Match*, or *Low Confidence*. +- **Existing Tracks:** Displays all scanned library files. Rows are highlighted when they are the best match for one or more incoming tracks. +- *Incoming Tracks* always contains all incoming files; “new” versus “matched” is indicated via status chips and filters rather than separate lists. + +### Detail Inspector +- Shows detailed information for the selected item, including tags, duration, bitrate (when available), format-based quality score, fingerprint distance, threshold used, and the single best-matched counterpart (lowest distance). If multiple potential matches exist, only the best candidate is actionable; others are informational. +- Optional shortcuts such as “open containing folder” and “play preview” are shown only when supported by the runtime environment. + +### Status Chips +- Each row may display one or more chips, including *New*, *Collision*, *Exact Match*, *Low Confidence*, *Potential Upgrade*, *Keep Existing*, or *Missing Metadata*. Chips are used consistently throughout the UI to avoid ambiguity. + +### Minimal Action Buttons (non-destructive only) +- **Filter: Collisions Only:** Toggles visibility to only matched items. +- **Sort: Quality Delta:** Sorts incoming tracks by quality difference relative to their matched library counterpart. +- **Flag for Copy:** Marks selected incoming tracks for inclusion in the export plan. +- **Flag for Replace:** Marks the incoming–existing pair (best match only) as a potential replacement candidate. +- **Export Plan / Report:** Generates a CSV or JSON summary of the current review state. +- **Clear Flags:** Removes all review flags without affecting scan results. +- No UI control, context menu, shortcut, or background task performs file moves, replacements, or playlist updates. + +### Progress and Status +- Separate progress indicators are shown for library and incoming scans. Each scan can be cancelled independently. A log panel displays scan configuration, counts, skipped files, and warnings. + +## Determining “Incoming” vs “Existing” +### Matching Logic +- Incoming tracks are compared against the existing library using an indexed fingerprint lookup rather than exhaustive pairwise comparison. Cached fingerprints are reused when available. + +### Thresholds +- Per-format thresholds take precedence over the global threshold. Distances below the active threshold are treated as matches; distances above are treated as non-matches. + +### Confidence Indicators +- **Exact Match:** Fingerprint distance equals zero. +- **Collision:** Distance is below the active threshold. +- **Low Confidence:** Distance is above the threshold but within a defined margin (e.g., threshold + fixed delta or percentage), prompting manual review. + +### Quality Comparison +- When a match exists, a quality score is computed using extension priority and bitrate when available. If bitrate is unavailable, extension priority alone is used. Incoming items with higher scores are labeled *Potential Upgrade*; lower-quality matches are labeled *Keep Existing*. + +## Error Handling and Edge Cases +- Scans support large libraries through incremental progress updates, cancellation after the current file completes processing, and reuse of cached fingerprints. +- Missing metadata is tolerated and flagged without blocking classification. +- Unsupported or unreadable files are skipped with logged reasons. +- Cancelled scans preserve partial results and are clearly labeled as *Partial*. +- Path and permission errors surface as inline banners and log entries without disabling retry. +- Threshold inputs are validated; invalid values are rejected with inline feedback and reverted to the last known good configuration. + +## Export Plan Schema (Minimum) +- Exported reports must include incoming path, existing path (nullable), match status, fingerprint distance, threshold used, quality scores, user flags, notes, timestamp, and a scan configuration hash. + +## Acceptance Criteria +- The UI contains exactly two primary lists: *Incoming Tracks* and *Existing Tracks*. +- No list, button, shortcut, or background process performs copy, replace, or playlist updates. +- Users can independently select folders, configure thresholds, and scan each source. +- Incoming tracks are clearly labeled via chips rather than separated into additional lists. +- Matching, quality comparison, and confidence indicators are visible and consistent. +- Exported reports fully represent the review state and require no file access to generate. +- Scan cancellation preserves completed results and logs without corrupting state. From ee89d92356ba6d08e359a920861c1b6a534fcab0 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Thu, 18 Dec 2025 20:15:16 -0500 Subject: [PATCH 124/606] Add review library sync entry point and persisted session models --- config.py | 4 + library_sync_review.py | 281 +++++++++++++++++++++++++++++++++++++++++ main_gui.py | 42 ++++++ 3 files changed, 327 insertions(+) create mode 100644 library_sync_review.py diff --git a/config.py b/config.py index 60958ab..ac9abad 100644 --- a/config.py +++ b/config.py @@ -60,6 +60,8 @@ def load_config(): cfg.setdefault("fingerprint_duration_ms", FP_DURATION_MS) cfg.setdefault("allow_mismatched_edits", ALLOW_MISMATCHED_EDITS) cfg.setdefault("library_root", "") + cfg.setdefault("use_library_sync_review", False) + cfg.setdefault("library_sync_review", {}) return cfg except Exception: return { @@ -69,6 +71,8 @@ def load_config(): "library_root": "", "duplicate_threshold": DEFAULT_DUP_THRESHOLD, "duplicate_prefix_len": DEFAULT_DUP_PREFIX_LEN, + "use_library_sync_review": False, + "library_sync_review": {}, } diff --git a/library_sync_review.py b/library_sync_review.py new file mode 100644 index 0000000..f99f67c --- /dev/null +++ b/library_sync_review.py @@ -0,0 +1,281 @@ +"""Review-first Library Sync entry point and session models.""" +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, Dict + +import tkinter as tk +from tkinter import filedialog, messagebox, ttk + +from config import DEFAULT_FP_THRESHOLDS, load_config, save_config + +REVIEW_CONFIG_KEY = "library_sync_review" +REVIEW_FLAG_KEY = "use_library_sync_review" +DEFAULT_REPORT_VERSION = 1 + + +class ScanState(str, Enum): + """States tracked during a review scan lifecycle.""" + + IDLE = "idle" + READY = "ready" + SCANNING_LIBRARY = "scanning_library" + SCANNING_INCOMING = "scanning_incoming" + COMPLETE = "complete" + CANCELLED = "cancelled" + + +@dataclass +class ScanConfig: + """Global and per-format fingerprint thresholds for a scan.""" + + global_threshold: float = DEFAULT_FP_THRESHOLDS.get("default", 0.3) + per_format_overrides: Dict[str, float] = field(default_factory=dict) + preset_name: str | None = None + + def to_dict(self) -> Dict[str, Any]: + return { + "global_threshold": float(self.global_threshold), + "per_format_overrides": dict(self.per_format_overrides), + "preset_name": self.preset_name, + } + + @classmethod + def from_dict(cls, data: Dict[str, Any] | None) -> "ScanConfig": + data = data or {} + overrides = data.get("per_format_overrides") or {} + return cls( + global_threshold=float(data.get("global_threshold", DEFAULT_FP_THRESHOLDS.get("default", 0.3))), + per_format_overrides={str(k): float(v) for k, v in overrides.items()}, + preset_name=data.get("preset_name"), + ) + + +@dataclass +class ScanSession: + """Persisted state for the review-first Library Sync experience.""" + + library_root: str = "" + incoming_root: str = "" + scan_config: ScanConfig = field(default_factory=ScanConfig) + scan_state: ScanState = ScanState.IDLE + exported_report_version: int = DEFAULT_REPORT_VERSION + + def to_dict(self) -> Dict[str, Any]: + return { + "library_root": self.library_root, + "incoming_root": self.incoming_root, + "scan_config": self.scan_config.to_dict(), + "scan_state": self.scan_state.value, + "exported_report_version": int(self.exported_report_version), + } + + @classmethod + def from_dict(cls, data: Dict[str, Any] | None) -> "ScanSession": + data = data or {} + state_value = data.get("scan_state") or ScanState.IDLE.value + try: + scan_state = ScanState(state_value) + except ValueError: + scan_state = ScanState.IDLE + return cls( + library_root=data.get("library_root", "") or "", + incoming_root=data.get("incoming_root", "") or "", + scan_config=ScanConfig.from_dict(data.get("scan_config")), + scan_state=scan_state, + exported_report_version=int(data.get("exported_report_version", DEFAULT_REPORT_VERSION)), + ) + + +def load_scan_session() -> ScanSession: + """Load the persisted review scan session from config.""" + cfg = load_config() + session = ScanSession.from_dict(cfg.get(REVIEW_CONFIG_KEY)) + cfg[REVIEW_CONFIG_KEY] = session.to_dict() + save_config(cfg) + return session + + +def save_scan_session(session: ScanSession) -> None: + """Persist the review scan session to config.""" + cfg = load_config() + cfg[REVIEW_CONFIG_KEY] = session.to_dict() + save_config(cfg) + + +def is_review_enabled() -> bool: + """Return whether the review-first entry point is enabled.""" + cfg = load_config() + return bool(cfg.get(REVIEW_FLAG_KEY, False)) + + +def set_review_enabled(enabled: bool) -> None: + """Toggle the review-first entry point flag.""" + cfg = load_config() + cfg[REVIEW_FLAG_KEY] = bool(enabled) + save_config(cfg) + + +class LibrarySyncReviewWindow(tk.Toplevel): + """Lightweight entry point for the redesign with persisted settings.""" + + def __init__(self, master: tk.Widget | None = None): + super().__init__(master) + self.title("Library Sync (Review)") + self.resizable(True, False) + self.session = load_scan_session() + + self.library_var = tk.StringVar(value=self.session.library_root) + self.incoming_var = tk.StringVar(value=self.session.incoming_root) + self.global_threshold_var = tk.StringVar(value=str(self.session.scan_config.global_threshold)) + self.preset_var = tk.StringVar(value=self.session.scan_config.preset_name or "") + self.report_version_var = tk.StringVar(value=str(self.session.exported_report_version)) + self.scan_state_var = tk.StringVar(value=self._describe_state(self.session.scan_state)) + + self._build_ui() + self.protocol("WM_DELETE_WINDOW", self._on_close) + + def _build_ui(self) -> None: + padding = {"padx": 10, "pady": 5} + + folder_frame = ttk.LabelFrame(self, text="Folders") + folder_frame.pack(fill="x", **padding) + + self._add_browse_row( + folder_frame, "Existing Library:", self.library_var, self._browse_library + ) + self._add_browse_row( + folder_frame, "Incoming Folder:", self.incoming_var, self._browse_incoming + ) + + scan_frame = ttk.LabelFrame(self, text="Scan Configuration") + scan_frame.pack(fill="x", **padding) + + thr_row = ttk.Frame(scan_frame) + thr_row.pack(fill="x", padx=5, pady=(5, 2)) + ttk.Label(thr_row, text="Global Threshold:").pack(side="left") + ttk.Entry(thr_row, textvariable=self.global_threshold_var, width=10).pack( + side="left", padx=(5, 0) + ) + + preset_row = ttk.Frame(scan_frame) + preset_row.pack(fill="x", padx=5, pady=2) + ttk.Label(preset_row, text="Preset Name:").pack(side="left") + ttk.Entry(preset_row, textvariable=self.preset_var, width=20).pack( + side="left", padx=(5, 0) + ) + + overrides_frame = ttk.Frame(scan_frame) + overrides_frame.pack(fill="both", expand=True, padx=5, pady=(2, 5)) + ttk.Label( + overrides_frame, + text="Per-format overrides (ext=threshold per line):", + ).pack(anchor="w") + self.overrides_text = tk.Text(overrides_frame, height=4) + self.overrides_text.pack(fill="x", expand=True, pady=(2, 0)) + self.overrides_text.insert("1.0", self._format_overrides()) + + meta_frame = ttk.LabelFrame(self, text="Session") + meta_frame.pack(fill="x", **padding) + meta_row = ttk.Frame(meta_frame) + meta_row.pack(fill="x", padx=5, pady=(5, 2)) + ttk.Label(meta_row, text="Scan State:").pack(side="left") + ttk.Label(meta_row, textvariable=self.scan_state_var).pack(side="left", padx=(5, 0)) + + version_row = ttk.Frame(meta_frame) + version_row.pack(fill="x", padx=5, pady=(2, 5)) + ttk.Label(version_row, text="Exported Report Version:").pack(side="left") + ttk.Entry(version_row, textvariable=self.report_version_var, width=6).pack( + side="left", padx=(5, 0) + ) + + btn_row = ttk.Frame(self) + btn_row.pack(fill="x", pady=(0, 10)) + ttk.Button(btn_row, text="Save Session", command=self._persist_session).pack( + side="left", padx=(10, 5) + ) + ttk.Button(btn_row, text="Close", command=self._on_close).pack(side="right", padx=(5, 10)) + + def _add_browse_row( + self, + parent: ttk.Frame, + label: str, + variable: tk.StringVar, + command, + ) -> None: + row = ttk.Frame(parent) + row.pack(fill="x", padx=5, pady=2) + ttk.Label(row, text=label).pack(side="left") + ttk.Entry(row, textvariable=variable).pack(side="left", fill="x", expand=True, padx=(5, 5)) + ttk.Button(row, text="Browse…", command=command).pack(side="left") + + def _browse_library(self) -> None: + folder = filedialog.askdirectory(title="Select Library Root", initialdir=self.library_var.get() or None) + if folder: + self.library_var.set(folder) + + def _browse_incoming(self) -> None: + folder = filedialog.askdirectory(title="Select Incoming Folder", initialdir=self.incoming_var.get() or None) + if folder: + self.incoming_var.set(folder) + + def _format_overrides(self) -> str: + lines = [] + for ext, thr in self.session.scan_config.per_format_overrides.items(): + lines.append(f"{ext}={thr}") + return "\n".join(lines) + + def _parse_overrides(self) -> Dict[str, float] | None: + overrides: Dict[str, float] = {} + raw = self.overrides_text.get("1.0", "end").strip() + if not raw: + return overrides + for line in raw.splitlines(): + if "=" not in line: + messagebox.showerror("Invalid Override", f"Expected 'ext=threshold' format, got: {line}") + return None + ext, value = line.split("=", 1) + try: + overrides[ext.strip()] = float(value.strip()) + except ValueError: + messagebox.showerror("Invalid Threshold", f"Could not parse threshold for {ext.strip()}: {value}") + return None + return overrides + + def _persist_session(self) -> bool: + overrides = self._parse_overrides() + if overrides is None: + return False + try: + global_thr = float(self.global_threshold_var.get()) + except ValueError: + messagebox.showerror("Invalid Threshold", "Global threshold must be a number.") + return False + try: + report_ver = int(self.report_version_var.get()) + except ValueError: + messagebox.showerror("Invalid Report Version", "Exported report version must be an integer.") + return False + + self.session = ScanSession( + library_root=self.library_var.get().strip(), + incoming_root=self.incoming_var.get().strip(), + scan_config=ScanConfig( + global_threshold=global_thr, + per_format_overrides=overrides, + preset_name=self.preset_var.get().strip() or None, + ), + scan_state=self.session.scan_state, + exported_report_version=report_ver, + ) + save_scan_session(self.session) + self.scan_state_var.set(self._describe_state(self.session.scan_state)) + return True + + def _describe_state(self, state: ScanState) -> str: + return state.value.replace("_", " ").title() + + def _on_close(self) -> None: + if self._persist_session(): + self.destroy() diff --git a/main_gui.py b/main_gui.py index faf6774..b430f4e 100644 --- a/main_gui.py +++ b/main_gui.py @@ -54,6 +54,7 @@ from typing import Any, Callable, List from indexer_control import cancel_event, IndexCancelled import library_sync +import library_sync_review import playlist_generator import crash_watcher @@ -1552,6 +1553,10 @@ def __init__(self): self.sync_improved = [] self.sync_existing_matches = [] self.sync_unmatched_new = [] + self.use_review_sync_var = tk.BooleanVar( + value=cfg.get("use_library_sync_review", False) + ) + self.sync_review_window: library_sync_review.LibrarySyncReviewWindow | None = None # ── Tag Fixer state ── self.tagfix_folder_var = tk.StringVar(value="") @@ -1660,6 +1665,15 @@ def build_ui(self): menubar.add_cascade(label="Settings", menu=settings_menu) tools_menu = tk.Menu(menubar, tearoff=False) + tools_menu.add_command( + label="Library Sync…", command=self._open_library_sync_tool + ) + tools_menu.add_checkbutton( + label="Use Library Sync (Review)", + variable=self.use_review_sync_var, + command=self._toggle_library_sync_mode, + ) + tools_menu.add_separator() tools_menu.add_command(label="Fix Tags via AcoustID", command=self.fix_tags_gui) tools_menu.add_command( label="Generate Library Index…", @@ -3566,6 +3580,34 @@ def reset_tagfix_log(self): self._log(f"Reset tag-fix log for {folder}") # ── Library Sync Helpers ─────────────────────────────────────────── + def _toggle_library_sync_mode(self) -> None: + cfg = load_config() + cfg["use_library_sync_review"] = bool(self.use_review_sync_var.get()) + save_config(cfg) + + def _open_library_sync_tool(self) -> None: + if self.use_review_sync_var.get(): + if self.sync_review_window and self.sync_review_window.winfo_exists(): + self.sync_review_window.lift() + self.sync_review_window.focus_set() + return + try: + self.sync_review_window = library_sync_review.LibrarySyncReviewWindow( + self + ) + self.sync_review_window.bind( + "", lambda _e: setattr(self, "sync_review_window", None) + ) + except Exception as exc: + messagebox.showerror("Library Sync (Review)", str(exc)) + else: + try: + self.notebook.select(self.sync_tab) + except Exception: + messagebox.showinfo( + "Library Sync", "The classic Library Sync tab is not available." + ) + def _browse_sync_library(self): initial = self.sync_library_var.get() or load_last_path() folder = filedialog.askdirectory( From 49760b4d3ac8bcb8588eb7983f3cc419969a2998 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Thu, 18 Dec 2025 22:00:38 -0500 Subject: [PATCH 125/606] Implement redesigned library sync scanning pipeline --- fingerprint_cache.py | 23 +- library_sync.py | 467 ++++++++++++++++++++++++++++--------- tests/test_library_sync.py | 53 +++-- 3 files changed, 415 insertions(+), 128 deletions(-) diff --git a/fingerprint_cache.py b/fingerprint_cache.py index 735314a..1f1460b 100644 --- a/fingerprint_cache.py +++ b/fingerprint_cache.py @@ -27,6 +27,7 @@ def _ensure_db(db_path: str) -> sqlite3.Connection: CREATE TABLE IF NOT EXISTS fingerprints ( path TEXT PRIMARY KEY, mtime REAL, + size INTEGER, duration INT, fingerprint TEXT ); @@ -39,6 +40,8 @@ def _ensure_db(db_path: str) -> sqlite3.Connection: } if "mtime" not in cols: conn.execute("ALTER TABLE fingerprints ADD COLUMN mtime REAL") + if "size" not in cols: + conn.execute("ALTER TABLE fingerprints ADD COLUMN size INTEGER") if "duration" not in cols: conn.execute("ALTER TABLE fingerprints ADD COLUMN duration INT") if "fingerprint" not in cols: @@ -64,16 +67,19 @@ def get_fingerprint( conn = _ensure_db(db_path) try: mtime = os.path.getmtime(path) + size = os.path.getsize(path) except OSError as e: log_callback(f"! Could not stat {path}: {e}") conn.close() return None row = conn.execute( - "SELECT mtime, fingerprint FROM fingerprints WHERE path=?", + "SELECT mtime, size, fingerprint FROM fingerprints WHERE path=?", (path,), ).fetchone() - if row and abs(row[0] - mtime) < 1e-6: - fp = row[1] + cached_mtime = row[0] if row else None + cached_size = row[1] if row else None + if row and abs(cached_mtime - mtime) < 1e-6 and int(cached_size or 0) == int(size): + fp = row[2] _dlog("FP", f"cache hit {path}", log_callback) conn.close() if isinstance(fp, (bytes, bytearray)): @@ -83,14 +89,20 @@ def get_fingerprint( fp = fp.decode("latin1", errors="ignore") _dlog("FP", f"fingerprint={fp} prefix={fp[:16]}", log_callback) return fp + if row: + _dlog( + "FP", + f"cache invalidated {path} stored_mtime={cached_mtime} stored_size={cached_size} new_mtime={mtime} new_size={size}", + log_callback, + ) _dlog("FP", f"cache miss {path}", log_callback) duration, fp_hash = compute_func(path) if fp_hash is not None: _dlog("FP", f"computed fingerprint {fp_hash} prefix={fp_hash[:16]}", log_callback) conn.execute( - "INSERT OR REPLACE INTO fingerprints (path, mtime, duration, fingerprint) VALUES (?, ?, ?, ?)", - (path, mtime, duration, fp_hash), + "INSERT OR REPLACE INTO fingerprints (path, mtime, size, duration, fingerprint) VALUES (?, ?, ?, ?, ?)", + (path, mtime, size, duration, fp_hash), ) conn.commit() conn.close() @@ -108,4 +120,3 @@ def flush_cache(db_path: str) -> None: conn.execute("DROP TABLE IF EXISTS fingerprints") conn.commit() conn.close() - diff --git a/library_sync.py b/library_sync.py index 80aede6..0e9f173 100644 --- a/library_sync.py +++ b/library_sync.py @@ -1,25 +1,32 @@ """Helpers for comparing an existing library to an incoming folder.""" from __future__ import annotations -import os +import hashlib import logging -from typing import Dict, List, Tuple, Callable, Optional +import os +from dataclasses import dataclass, field +from enum import Enum +from typing import Callable, Dict, Iterable, List, Optional, Tuple import music_indexer_api as idx import fingerprint_generator from fingerprint_cache import _ensure_db from near_duplicate_detector import fingerprint_distance -from config import NEAR_DUPLICATE_THRESHOLD, FORMAT_PRIORITY +from config import FORMAT_PRIORITY, DEFAULT_FP_THRESHOLDS from crash_watcher import record_event from crash_logger import watcher - debug: bool = False _logger = logging.getLogger(__name__) _logger.propagate = False _logger.addHandler(logging.NullHandler()) _file_handler: Optional[logging.Handler] = None +NEAR_MISS_MARGIN_RATIO = 0.25 +NEAR_MISS_MARGIN_FLOOR = 0.02 +SIGNATURE_TOKEN_COUNT = 8 +SHORTLIST_FALLBACK = 50 + def set_debug(enabled: bool, log_root: str | None = None) -> None: """Enable or disable verbose debug logging. @@ -56,23 +63,162 @@ def _dlog(msg: str, log_callback: Callable[[str], None] | None = None) -> None: log_callback(msg) _logger.debug(msg) -@watcher.traced + +def _normalize_path(path: str) -> str: + return os.path.normcase(os.path.normpath(os.path.abspath(path))) + + +def _stable_track_id(path: str) -> str: + norm = _normalize_path(path) + return hashlib.sha1(norm.encode("utf-8")).hexdigest() + + +class MatchStatus(str, Enum): + NEW = "new" + COLLISION = "collision" + EXACT_MATCH = "exact_match" + LOW_CONFIDENCE = "low_confidence" + + +@dataclass +class TrackRecord: + """Normalized representation of a scanned track.""" + + track_id: str + path: str + normalized_path: str + ext: str + bitrate: int | None + size: int + mtime: float + fingerprint: str | None + tags: Dict[str, object] + duration: int | None = None + + def signature(self) -> str | None: + return _fingerprint_signature(self.fingerprint) + + def to_dict(self) -> Dict[str, object]: + return { + "track_id": self.track_id, + "path": self.path, + "normalized_path": self.normalized_path, + "ext": self.ext, + "bitrate": self.bitrate, + "size": self.size, + "mtime": self.mtime, + "fingerprint": self.fingerprint, + "duration": self.duration, + "tags": dict(self.tags), + } + + +@dataclass(order=True) +class CandidateDistance: + """Distance info for a shortlist candidate.""" + + distance: float + track: TrackRecord = field(compare=False) + + def to_dict(self) -> Dict[str, object]: + return {"track_id": self.track.track_id, "path": self.track.path, "distance": self.distance} + + +@dataclass +class MatchResult: + """Matching outcome for a single incoming track.""" + + incoming: TrackRecord + existing: TrackRecord | None + status: MatchStatus + distance: float | None + threshold_used: float + near_miss_margin: float + confidence: float + candidates: List[CandidateDistance] + quality_label: str | None + incoming_score: int + existing_score: int | None + + def to_dict(self) -> Dict[str, object]: + return { + "incoming": self.incoming.to_dict(), + "existing": self.existing.to_dict() if self.existing else None, + "status": self.status.value, + "distance": self.distance, + "threshold_used": self.threshold_used, + "near_miss_margin": self.near_miss_margin, + "confidence": self.confidence, + "candidates": [c.to_dict() for c in self.candidates], + "quality_label": self.quality_label, + "incoming_score": self.incoming_score, + "existing_score": self.existing_score, + } + + +def _fingerprint_signature(fp: str | None) -> str | None: + if not fp: + return None + tokens = fp.replace(",", " ").split() + if not tokens: + return None + return " ".join(tokens[:SIGNATURE_TOKEN_COUNT]) + + +def _select_threshold(ext: str, thresholds: Dict[str, float]) -> float: + return float(thresholds.get(ext, thresholds.get("default", 0.3))) + + +def compute_quality_score(info: Dict[str, object], fmt_priority: Dict[str, int]) -> int: + pr = fmt_priority.get(info.get("ext"), 0) + bitrate_val = info.get("bitrate") + br = int(bitrate_val) if bitrate_val not in (None, "", False) else 0 + score = pr * br if br else pr + _dlog( + f"DEBUG: Quality score ext={info.get('ext')} priority={pr} bitrate={br} score={score}", + None, + ) + return score + + +def _compute_confidence(distance: float | None, threshold: float, margin: float) -> float: + if distance is None: + return 0.0 + if distance <= 0: + return 1.0 + limit = threshold + margin + if limit <= 0: + return 0.0 + score = max(0.0, min(1.0, 1 - (distance / limit))) + return score + + def _scan_folder( folder: str, db_path: str, + on_record: Callable[[TrackRecord], None] | None = None, log_callback: Callable[[str], None] | None = None, -) -> Dict[str, Dict[str, object]]: + progress_callback: Callable[[int, int, str, str], None] | None = None, +) -> List[TrackRecord]: + """Enumerate audio files, normalize paths, and attach cached fingerprints.""" record_event(f"library_sync: scanning folder {folder}") _dlog(f"DEBUG: Scanning folder {folder}", log_callback) - infos: Dict[str, Dict[str, object]] = {} - audio_paths: List[str] = [] + entries: List[Dict[str, object]] = [] + total_files = 0 for dirpath, _dirs, files in os.walk(folder): for fname in files: ext = os.path.splitext(fname)[1].lower() if ext not in idx.SUPPORTED_EXTS: continue path = os.path.join(dirpath, fname) - _dlog(f"DEBUG: Processing {path}", log_callback) + norm = _normalize_path(path) + try: + stat = os.stat(path) + mtime = stat.st_mtime + size = stat.st_size + except OSError as e: + _dlog(f"DEBUG: Skipping unreadable file {path}: {e}", log_callback) + continue tags = idx.get_tags(path) bitrate = 0 try: @@ -80,56 +226,195 @@ def _scan_folder( if audio and getattr(audio, "info", None): bitrate = getattr(audio.info, "bitrate", 0) or getattr(audio.info, "sample_rate", 0) or 0 except Exception: - pass - infos[path] = { - **tags, - "fp": None, - "ext": ext, - "bitrate": bitrate, - } - audio_paths.append(path) - - if not audio_paths: - return infos + bitrate = 0 + entries.append( + { + "path": path, + "normalized_path": norm, + "ext": ext, + "bitrate": bitrate, + "tags": tags, + "mtime": mtime, + "size": size, + } + ) + total_files += 1 + if progress_callback: + progress_callback(total_files, 0, path, "scan") + + if not entries: + return [] conn = _ensure_db(db_path) + to_compute: List[str] = [] + entry_map = {e["path"]: e for e in entries} + + for entry in entries: + row = conn.execute( + "SELECT mtime, size, duration, fingerprint FROM fingerprints WHERE path=?", + (entry["path"],), + ).fetchone() + cached_mtime = row[0] if row else None + cached_size = row[1] if row else None + if row and abs(cached_mtime - entry["mtime"]) < 1e-6 and int(cached_size or 0) == int(entry["size"]): + entry["duration"] = row[2] + entry["fingerprint"] = row[3] + _dlog(f"DEBUG: cache hit {entry['path']}", log_callback) + else: + if row: + msg = ( + "DEBUG: cache invalidation " + f"path={entry['path']} stored_mtime={cached_mtime} stored_size={cached_size} " + f"new_mtime={entry['mtime']} new_size={entry['size']}" + ) + if log_callback: + log_callback(msg) + _dlog(msg, log_callback) + to_compute.append(entry["path"]) + + def fp_progress(current: int, total: int, path: str, phase: str) -> None: + if progress_callback: + progress_callback(current, total, path, phase) + + if to_compute: + for result in fingerprint_generator.compute_fingerprints_parallel( + to_compute, db_path, log_callback, fp_progress + ): + path = result[0] + duration = result[1] if len(result) > 2 else None + fp = result[-1] + entry = entry_map.get(path) + if entry is None: + continue + entry["fingerprint"] = fp + entry["duration"] = duration + conn.execute( + "INSERT OR REPLACE INTO fingerprints (path, mtime, size, duration, fingerprint) VALUES (?, ?, ?, ?, ?)", + (path, entry["mtime"], entry["size"], duration, fp), + ) + conn.commit() + conn.close() - def progress_cb(current: int, total: int, path: str, _phase: str) -> None: - if log_callback and path: - log_callback(f"{current}/{total}: {path}") - - for result in fingerprint_generator.compute_fingerprints_parallel( - audio_paths, db_path, log_callback, progress_cb - ): - path = result[0] - fp = result[-1] - duration = result[1] if len(result) > 2 else None - try: - mtime = os.path.getmtime(path) - except OSError: - mtime = 0.0 - conn.execute( - "INSERT OR REPLACE INTO fingerprints (path, mtime, duration, fingerprint) VALUES (?, ?, ?, ?)", - (path, mtime, duration, fp), + records: List[TrackRecord] = [] + for entry in entries: + rec = TrackRecord( + track_id=_stable_track_id(entry["normalized_path"]), + path=entry["path"], + normalized_path=entry["normalized_path"], + ext=entry["ext"], + bitrate=int(entry.get("bitrate") or 0), + size=int(entry.get("size") or 0), + mtime=float(entry.get("mtime") or 0.0), + fingerprint=entry.get("fingerprint"), + tags=entry.get("tags", {}), + duration=entry.get("duration"), ) - if path in infos: - infos[path]["fp"] = fp + records.append(rec) + if on_record: + on_record(rec) - conn.commit() - conn.close() record_event(f"library_sync: finished scanning {folder}") - return infos + return records -def compute_quality_score(info: Dict[str, object], fmt_priority: Dict[str, int]) -> int: - pr = fmt_priority.get(info.get("ext"), 0) - br = int(info.get("bitrate") or 0) - score = pr * br +def _build_candidate_index(records: Iterable[TrackRecord]) -> Tuple[Dict[str, List[TrackRecord]], Dict[str, List[TrackRecord]]]: + sig_index: Dict[str, List[TrackRecord]] = {} + ext_index: Dict[str, List[TrackRecord]] = {} + for rec in records: + sig = rec.signature() + if sig: + sig_index.setdefault(sig, []).append(rec) + ext_index.setdefault(rec.ext, []).append(rec) + return sig_index, ext_index + + +def _shortlist_candidates( + incoming: TrackRecord, + sig_index: Dict[str, List[TrackRecord]], + ext_index: Dict[str, List[TrackRecord]], + fallback_pool: List[TrackRecord], + log_callback: Callable[[str], None] | None = None, +) -> List[TrackRecord]: + sig = incoming.signature() + candidates: List[TrackRecord] = [] + if sig and sig in sig_index: + candidates.extend(sig_index[sig]) + if not candidates: + candidates.extend(ext_index.get(incoming.ext, [])[:SHORTLIST_FALLBACK]) + if not candidates and fallback_pool: + candidates.extend(fallback_pool[:SHORTLIST_FALLBACK]) _dlog( - f"DEBUG: Quality score ext={info.get('ext')} priority={pr} bitrate={br} score={score}", - None, + f"DEBUG: shortlist for {incoming.path} -> {len(candidates)} candidates", + log_callback, ) - return score + return candidates + + +def _match_tracks( + incoming: List[TrackRecord], + library: List[TrackRecord], + thresholds: Dict[str, float], + fmt_priority: Dict[str, int], + log_callback: Callable[[str], None] | None = None, +) -> List[MatchResult]: + sig_index, ext_index = _build_candidate_index(library) + fallback_pool = sorted([r for r in library if r.fingerprint], key=lambda r: r.normalized_path) + results: List[MatchResult] = [] + for inc in incoming: + threshold_used = _select_threshold(inc.ext, thresholds) + near_miss_margin = max(NEAR_MISS_MARGIN_FLOOR, threshold_used * NEAR_MISS_MARGIN_RATIO) + candidates = _shortlist_candidates(inc, sig_index, ext_index, fallback_pool, log_callback=log_callback) + cand_dist: List[CandidateDistance] = [] + for cand in candidates: + dist = fingerprint_distance(inc.fingerprint, cand.fingerprint) + cand_dist.append(CandidateDistance(distance=dist, track=cand)) + cand_dist.sort() + best = cand_dist[0] if cand_dist else None + best_track = best.track if best else None + best_distance = best.distance if best else None + + if best_distance is None or best_track is None: + status = MatchStatus.NEW + elif best_distance == 0: + status = MatchStatus.EXACT_MATCH + elif best_distance <= threshold_used: + status = MatchStatus.COLLISION + elif best_distance <= threshold_used + near_miss_margin: + status = MatchStatus.LOW_CONFIDENCE + else: + status = MatchStatus.NEW + + inc_score = compute_quality_score(inc.__dict__, fmt_priority) + existing_score = compute_quality_score(best_track.__dict__, fmt_priority) if best_track else None + if best_track is None: + quality_label = None + elif inc_score > (existing_score or 0): + quality_label = "Potential Upgrade" + else: + quality_label = "Keep Existing" + + confidence = _compute_confidence(best_distance, threshold_used, near_miss_margin) + _dlog( + f"DEBUG: match incoming={inc.path} best={best_track.path if best_track else 'none'} " + f"dist={best_distance} thr={threshold_used} margin={near_miss_margin} status={status} confidence={confidence}", + log_callback, + ) + results.append( + MatchResult( + incoming=inc, + existing=best_track, + status=status, + distance=best_distance, + threshold_used=threshold_used, + near_miss_margin=near_miss_margin, + confidence=confidence, + candidates=cand_dist, + quality_label=quality_label, + incoming_score=inc_score, + existing_score=existing_score, + ) + ) + return results @watcher.traced @@ -141,9 +426,13 @@ def compare_libraries( fmt_priority: Dict[str, int] | None = None, thresholds: Dict[str, float] | None = None, log_callback: Callable[[str], None] | None = None, + progress_callback: Callable[[int, int, str, str], None] | None = None, ) -> Dict[str, object]: """Return classification of incoming files vs. an existing library. + Scans are performed independently so the UI can remain responsive, and + matching uses indexed shortlist candidates to avoid pairwise explosion. + Parameters ---------- thresholds: @@ -157,82 +446,44 @@ def compare_libraries( if threshold is not None: thresholds = {"default": threshold} else: - from config import DEFAULT_FP_THRESHOLDS - thresholds = DEFAULT_FP_THRESHOLDS fmt_priority = fmt_priority or FORMAT_PRIORITY - lib_infos = _scan_folder(library_folder, db_path, log_callback) - inc_infos = _scan_folder(incoming_folder, db_path, log_callback) - existing_tracks: List[str] = list(lib_infos.keys()) - new_tracks: List[str] = list(inc_infos.keys()) + lib_records = _scan_folder(library_folder, db_path, log_callback=log_callback, progress_callback=progress_callback) + inc_records = _scan_folder(incoming_folder, db_path, log_callback=log_callback, progress_callback=progress_callback) - new: List[str] = [] + match_results = _match_tracks(inc_records, lib_records, thresholds, fmt_priority, log_callback=log_callback) + + new_paths: List[str] = [] existing_matches: List[Tuple[str, str]] = [] improved: List[Tuple[str, str]] = [] - for inc_path, inc_info in inc_infos.items(): - _dlog(f"DEBUG: Comparing incoming {inc_path}", log_callback) - best_match = None - best_dist = 1.0 - for lib_path, lib_info in lib_infos.items(): - dist = fingerprint_distance(inc_info.get("fp"), lib_info.get("fp")) - _dlog( - f"DEBUG: Distance {inc_path} -> {lib_path}: {dist:.4f}", - log_callback, - ) - if dist == 0: - best_match = lib_path - best_dist = 0 - break - if dist < best_dist: - best_match = lib_path - best_dist = dist - if best_match is None: - _dlog("DEBUG: No match found", log_callback) - new.append(inc_path) + for res in match_results: + if res.status == MatchStatus.NEW: + new_paths.append(res.incoming.path) continue - best_ext = lib_infos[best_match]["ext"] - thr = thresholds.get(best_ext, thresholds.get("default", 0.3)) - _dlog( - f"DEBUG: Best distance {best_dist:.4f} threshold {thr:.4f}", - log_callback, - ) - if best_dist >= thr: - _dlog("DEBUG: Above threshold, marked as new", log_callback) - new.append(inc_path) + if res.existing is None: + new_paths.append(res.incoming.path) continue - inc_score = compute_quality_score(inc_info, fmt_priority) - lib_score = compute_quality_score(lib_infos[best_match], fmt_priority) - _dlog( - f"DEBUG: Quality inc={inc_score} lib={lib_score}", - log_callback, - ) - if inc_score > lib_score: - _dlog("DEBUG: Incoming is higher quality", log_callback) - improved.append((inc_path, best_match)) + if res.quality_label == "Potential Upgrade": + improved.append((res.incoming.path, res.existing.path)) else: - _dlog("DEBUG: Existing is higher quality", log_callback) - existing_matches.append((inc_path, best_match)) + existing_matches.append((res.incoming.path, res.existing.path)) - _dlog( - "DEBUG: Compare complete " - f"library_existing={len(existing_tracks)} " - f"incoming_tracks={len(new_tracks)} " - f"new={len(new)} existing_matches={len(existing_matches)} improved={len(improved)}", - log_callback, - ) result = { - "existing": existing_tracks, - "new_tracks": new_tracks, - "new": new, + "existing": [r.path for r in lib_records], + "new_tracks": [r.path for r in inc_records], + "new": new_paths, "existing_matches": existing_matches, "improved": improved, + "library_records": [r.to_dict() for r in lib_records], + "incoming_records": [r.to_dict() for r in inc_records], + "matches": [m.to_dict() for m in match_results], } record_event( "library_sync: comparison complete " - f"library_existing={len(existing_tracks)} incoming_tracks={len(new_tracks)} " - f"new={len(new)} existing_matches={len(existing_matches)} improved={len(improved)}" + f"library_existing={len(lib_records)} incoming_tracks={len(inc_records)} " + f"new={len(new_paths)} existing_matches={len(existing_matches)} improved={len(improved)}" ) return result diff --git a/tests/test_library_sync.py b/tests/test_library_sync.py index 355fe62..0f3b0f5 100644 --- a/tests/test_library_sync.py +++ b/tests/test_library_sync.py @@ -39,7 +39,7 @@ def fake_compute(paths, db_path, log_callback=None, progress_callback=None): importlib.reload(music_indexer_api) import library_sync importlib.reload(library_sync) -from library_sync import compare_libraries, compute_quality_score +from library_sync import compare_libraries, compute_quality_score, MatchStatus from fingerprint_cache import flush_cache @@ -47,6 +47,8 @@ def test_quality_score_simple(): info = {'ext': '.flac', 'bitrate': 1000} score = compute_quality_score(info, {'.flac': 3, '.mp3': 1}) assert score == 3000 + fallback = compute_quality_score({"ext": ".mp3", "bitrate": None}, {".mp3": 2}) + assert fallback == 2 def test_compare_libraries(tmp_path): @@ -63,46 +65,69 @@ def test_compare_libraries(tmp_path): db = tmp_path / 'fp.db' res = compare_libraries(str(lib), str(inc), str(db)) - existing_paths = set(res['existing']) + existing_paths = set(res["existing"]) assert existing_paths == {str(lib / 'a.flac'), str(lib / 'b.mp3')} - incoming_paths = set(res['new_tracks']) + incoming_paths = set(res["new_tracks"]) assert incoming_paths == { str(inc / 'a.flac'), str(inc / 'b.flac'), str(inc / 'new.mp3'), } - unmatched_new = set(res['new']) - assert str(inc / 'new.mp3') in unmatched_new + new_incoming = set(res["new"]) + assert str(inc / 'new.mp3') in new_incoming - ex_pairs = set(res['existing_matches']) - assert (str(inc / 'a.flac'), str(lib / 'a.flac')) in ex_pairs + existing_pairs = set(res["existing_matches"]) + assert (str(inc / 'a.flac'), str(lib / 'a.flac')) in existing_pairs - imp_pairs = set(res['improved']) - assert (str(inc / 'b.flac'), str(lib / 'b.mp3')) in imp_pairs + upgrades = set(res["improved"]) + assert (str(inc / 'b.flac'), str(lib / 'b.mp3')) in upgrades + + matches = {m["incoming"]["path"]: m for m in res["matches"]} + assert matches[str(inc / "a.flac")]["status"] == MatchStatus.EXACT_MATCH.value + assert matches[str(inc / "b.flac")]["quality_label"] == "Potential Upgrade" + assert matches[str(inc / "new.mp3")]["status"] == MatchStatus.NEW.value # cleanup cache between tests flush_cache(str(db)) -def test_compare_libraries_thresholds(tmp_path): +def test_compare_libraries_thresholds(tmp_path, monkeypatch): lib = tmp_path / 'lib' inc = tmp_path / 'inc' lib.mkdir() inc.mkdir() - (lib / 'a.flac').write_text('x') - (inc / 'a.flac').write_text('x') + (lib / 'loose.wav').write_text('x') + (inc / 'loose.wav').write_text('x') + + def fake_fp(path, **kw): + if "inc" in path: + return "1 1 1 1 1 1 1 2" + return "1 1 1 1 1 1 1 1" + + def fake_compute(paths, db_path, log_callback=None, progress_callback=None): + for p in paths: + fp = fake_fp(p) + yield p, 0, fp + + monkeypatch.setattr( + sys.modules["fingerprint_generator"], + "compute_fingerprints_parallel", + fake_compute, + ) db = tmp_path / 'fp.db' res = compare_libraries( str(lib), str(inc), str(db), - thresholds={"default": 0.0}, + thresholds={".wav": 0.1, "default": 0.3}, ) - assert str(inc / 'a.flac') in set(res['new']) + match = res["matches"][0] + assert match["threshold_used"] == 0.1 + assert match["status"] == MatchStatus.LOW_CONFIDENCE.value flush_cache(str(db)) From 40cae566dd70a42e5fa1a67618a7af5786b36708 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sat, 20 Dec 2025 10:43:23 -0500 Subject: [PATCH 126/606] Add review flag store and filtering helpers --- library_sync_review_state.py | 102 ++++++++++++++++++ tests/test_library_sync_review_state.py | 132 ++++++++++++++++++++++++ 2 files changed, 234 insertions(+) create mode 100644 library_sync_review_state.py create mode 100644 tests/test_library_sync_review_state.py diff --git a/library_sync_review_state.py b/library_sync_review_state.py new file mode 100644 index 0000000..73f47da --- /dev/null +++ b/library_sync_review_state.py @@ -0,0 +1,102 @@ +"""Review state storage and pure helpers for the Library Sync redesign.""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Dict, Iterable, List + +from library_sync import MatchResult, MatchStatus + + +@dataclass +class ReviewFlags: + """User selections captured during review.""" + + copy: set[str] = field(default_factory=set) + replace: Dict[str, str] = field(default_factory=dict) + + +class ReviewStateStore: + """In-memory store for review flags that survives view transforms.""" + + def __init__(self) -> None: + self.flags = ReviewFlags() + + def flag_for_copy(self, incoming_track_id: str) -> None: + """Mark an incoming row for copy.""" + self.flags.copy.add(incoming_track_id) + + def unflag_copy(self, incoming_track_id: str) -> None: + """Remove a copy flag from an incoming row.""" + self.flags.copy.discard(incoming_track_id) + + def is_copy_flagged(self, incoming_track_id: str) -> bool: + """Return whether the incoming row is flagged for copy.""" + return incoming_track_id in self.flags.copy + + def flag_for_replace(self, match: MatchResult) -> None: + """Flag the best-match pair for replacement.""" + if match.existing is None: + raise ValueError("Cannot flag for replace without a best match.") + self.flags.replace[match.incoming.track_id] = match.existing.track_id + + def replace_target(self, incoming_track_id: str) -> str | None: + """Return the flagged replacement target, if any.""" + return self.flags.replace.get(incoming_track_id) + + def clear_all(self) -> None: + """Remove all flags across the review session.""" + self.flags.copy.clear() + self.flags.replace.clear() + + def reconcile_best_matches(self, results: Iterable[MatchResult]) -> List[str]: + """Rebind replacement flags when the best match changes. + + Returns a list of warning messages describing any rebinding or clears. + """ + best_map = { + res.incoming.track_id: res.existing.track_id if res.existing else None for res in results + } + warnings: List[str] = [] + for incoming_id, existing_id in list(self.flags.replace.items()): + best_existing = best_map.get(incoming_id) + if best_existing is None: + warnings.append( + f"Cleared replace flag for {incoming_id} because it no longer has a best match." + ) + del self.flags.replace[incoming_id] + continue + if best_existing != existing_id: + warnings.append( + f"Rebound replace flag for {incoming_id} from {existing_id} to {best_existing} " + "after best match changed." + ) + self.flags.replace[incoming_id] = best_existing + return warnings + + +def filter_collisions_only(results: Iterable[MatchResult]) -> List[MatchResult]: + """Return only matched rows (collisions, exact matches, and near-misses).""" + return [res for res in results if res.existing is not None and res.status != MatchStatus.NEW] + + +def quality_delta(result: MatchResult) -> int | None: + """Compute quality difference between incoming and existing tracks.""" + if result.existing_score is None: + return None + return result.incoming_score - (result.existing_score or 0) + + +def sort_by_quality_delta(results: Iterable[MatchResult], descending: bool = True) -> List[MatchResult]: + """Sort incoming tracks by quality delta while keeping unmatched rows last.""" + indexed = list(enumerate(results)) + + def _key(item: tuple[int, MatchResult]) -> tuple[bool, float, int]: + idx, res = item + delta = quality_delta(res) + if delta is None: + return True, float("inf"), idx + if descending: + return False, -float(delta), idx + return False, float(delta), idx + + return [res for _idx, res in sorted(indexed, key=_key)] diff --git a/tests/test_library_sync_review_state.py b/tests/test_library_sync_review_state.py new file mode 100644 index 0000000..ffca2c8 --- /dev/null +++ b/tests/test_library_sync_review_state.py @@ -0,0 +1,132 @@ +import sys +import types + +# Stub heavy dependencies before importing modules under test +acoustid_stub = types.ModuleType("acoustid") +acoustid_stub.fingerprint_file = lambda path: (0, "stub-fp") +sys.modules.setdefault("acoustid", acoustid_stub) + +silence_stub = types.ModuleType("pydub.silence") +silence_stub.detect_nonsilent = lambda *args, **kwargs: [] + +audio_segment = type("AudioSegment", (), {"from_file": staticmethod(lambda path: None)}) +pydub_stub = types.ModuleType("pydub") +pydub_stub.AudioSegment = audio_segment +pydub_stub.silence = silence_stub + +sys.modules.setdefault("pydub", pydub_stub) +sys.modules.setdefault("pydub.silence", silence_stub) + +from library_sync import MatchResult, MatchStatus, TrackRecord +from library_sync_review_state import ( + ReviewStateStore, + filter_collisions_only, + quality_delta, + sort_by_quality_delta, +) + + +def _track(track_id: str, path: str, ext: str = ".mp3", bitrate: int = 192) -> TrackRecord: + return TrackRecord( + track_id=track_id, + path=path, + normalized_path=path, + ext=ext, + bitrate=bitrate, + size=1, + mtime=0.0, + fingerprint="fp", + tags={}, + duration=60, + ) + + +def _match( + incoming_id: str, + existing_id: str | None, + status: MatchStatus = MatchStatus.COLLISION, + incoming_score: int = 10, + existing_score: int | None = 5, +) -> MatchResult: + incoming = _track(incoming_id, f"/incoming/{incoming_id}") + existing = _track(existing_id, f"/library/{existing_id}") if existing_id else None + return MatchResult( + incoming=incoming, + existing=existing, + status=status, + distance=0.1 if existing else None, + threshold_used=0.3, + near_miss_margin=0.05, + confidence=0.8 if existing else 0.0, + candidates=[], + quality_label="Potential Upgrade" if existing_score is not None and incoming_score > existing_score else "Keep Existing", + incoming_score=incoming_score, + existing_score=existing_score, + ) + + +def test_collisions_filter_excludes_new_rows() -> None: + results = [ + _match("inc-new", None, status=MatchStatus.NEW, existing_score=None), + _match("inc-match", "lib-a", status=MatchStatus.COLLISION), + _match("inc-exact", "lib-b", status=MatchStatus.EXACT_MATCH), + ] + filtered = filter_collisions_only(results) + assert len(filtered) == 2 + assert all(res.existing is not None for res in filtered) + + +def test_quality_delta_sort_orders_by_upgrade() -> None: + upgrade = _match("inc-upgrade", "lib-low", incoming_score=12, existing_score=4) + downgrade = _match("inc-downgrade", "lib-high", incoming_score=5, existing_score=15) + unmatched = _match("inc-new", None, status=MatchStatus.NEW, existing_score=None) + sorted_results = sort_by_quality_delta([unmatched, downgrade, upgrade]) + assert sorted_results[0].incoming.track_id == upgrade.incoming.track_id + assert sorted_results[-1].incoming.track_id == unmatched.incoming.track_id + assert quality_delta(upgrade) == 8 + assert quality_delta(downgrade) == -10 + assert quality_delta(unmatched) is None + + +def test_replace_flags_rebind_when_best_match_changes() -> None: + store = ReviewStateStore() + first_result = _match("inc-1", "lib-1") + store.flag_for_replace(first_result) + + updated_results = [ + _match("inc-1", "lib-2", status=MatchStatus.COLLISION), + _match("inc-2", None, status=MatchStatus.NEW, existing_score=None), + ] + warnings = store.reconcile_best_matches(updated_results) + + assert store.replace_target("inc-1") == "lib-2" + assert len(warnings) == 1 + assert "lib-1" in warnings[0] and "lib-2" in warnings[0] + + +def test_replace_flags_clear_when_best_match_missing() -> None: + store = ReviewStateStore() + first_result = _match("inc-1", "lib-1") + store.flag_for_replace(first_result) + + warnings = store.reconcile_best_matches([_match("inc-1", None, status=MatchStatus.NEW, existing_score=None)]) + + assert store.replace_target("inc-1") is None + assert len(warnings) == 1 + assert "Cleared replace flag" in warnings[0] + + +def test_flags_survive_filtering_and_sorting() -> None: + store = ReviewStateStore() + flagged_copy = _match("inc-copy", "lib-copy") + flagged_replace = _match("inc-replace", "lib-replace") + + store.flag_for_copy(flagged_copy.incoming.track_id) + store.flag_for_replace(flagged_replace) + + filtered = filter_collisions_only([flagged_copy, flagged_replace]) + sorted_results = sort_by_quality_delta(filtered) + + assert len(sorted_results) == 2 + assert store.is_copy_flagged(flagged_copy.incoming.track_id) + assert store.replace_target(flagged_replace.incoming.track_id) == flagged_replace.existing.track_id From f5efb141d129de7d93337b874fa357b037e5f512 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sat, 20 Dec 2025 11:00:10 -0500 Subject: [PATCH 127/606] Add export reporting for library sync review --- library_sync_review.py | 62 ++++++- library_sync_review_report.py | 226 +++++++++++++++++++++++ library_sync_review_state.py | 13 ++ tests/test_library_sync_review_report.py | 178 ++++++++++++++++++ 4 files changed, 478 insertions(+), 1 deletion(-) create mode 100644 library_sync_review_report.py create mode 100644 tests/test_library_sync_review_report.py diff --git a/library_sync_review.py b/library_sync_review.py index f99f67c..5dc8ffd 100644 --- a/library_sync_review.py +++ b/library_sync_review.py @@ -9,10 +9,10 @@ from tkinter import filedialog, messagebox, ttk from config import DEFAULT_FP_THRESHOLDS, load_config, save_config +from library_sync_review_report import DEFAULT_REPORT_VERSION REVIEW_CONFIG_KEY = "library_sync_review" REVIEW_FLAG_KEY = "use_library_sync_review" -DEFAULT_REPORT_VERSION = 1 class ScanState(str, Enum): @@ -117,6 +117,62 @@ def set_review_enabled(enabled: bool) -> None: save_config(cfg) +class ReportPreviewDialog(tk.Toplevel): + """Simple dialog that summarizes report counts before export.""" + + LABELS = { + "new": "New", + "collisions": "Collisions / Exact Matches", + "low_confidence": "Low Confidence", + "flagged_copy": "Flagged Copy", + "flagged_replace": "Flagged Replace", + } + + def __init__(self, master: tk.Misc | None, summary: dict[str, int]): + super().__init__(master) + self.title("Report Preview") + self.resizable(False, False) + self.result = False + self.transient(master) + self.grab_set() + + ttk.Label(self, text="Review counts before saving the export report:").pack( + anchor="w", padx=10, pady=(10, 5) + ) + + for key, label in self.LABELS.items(): + row = ttk.Frame(self) + row.pack(fill="x", padx=10, pady=2) + ttk.Label(row, text=f"{label}:").pack(side="left") + ttk.Label(row, text=str(summary.get(key, 0)), font=("TkDefaultFont", 10, "bold")).pack( + side="right" + ) + + btn_row = ttk.Frame(self) + btn_row.pack(fill="x", padx=10, pady=(10, 10)) + ttk.Button(btn_row, text="Save Report", command=self._confirm).pack(side="left") + ttk.Button(btn_row, text="Cancel", command=self._cancel).pack(side="right") + + self.protocol("WM_DELETE_WINDOW", self._cancel) + + @classmethod + def prompt(cls, master: tk.Misc | None, summary: dict[str, int]) -> bool: + dialog = cls(master, summary) + if master is not None: + master.wait_window(dialog) + else: + dialog.wait_window() + return bool(getattr(dialog, "result", False)) + + def _confirm(self) -> None: + self.result = True + self.destroy() + + def _cancel(self) -> None: + self.result = False + self.destroy() + + class LibrarySyncReviewWindow(tk.Toplevel): """Lightweight entry point for the redesign with persisted settings.""" @@ -276,6 +332,10 @@ def _persist_session(self) -> bool: def _describe_state(self, state: ScanState) -> str: return state.value.replace("_", " ").title() + def preview_report(self, summary: dict[str, int]) -> bool: + """Open a modal preview dialog to confirm report export counts.""" + return ReportPreviewDialog.prompt(self, summary) + def _on_close(self) -> None: if self._persist_session(): self.destroy() diff --git a/library_sync_review_report.py b/library_sync_review_report.py new file mode 100644 index 0000000..9377635 --- /dev/null +++ b/library_sync_review_report.py @@ -0,0 +1,226 @@ +"""Export and reporting helpers for the Library Sync review experience.""" +from __future__ import annotations + +import csv +import hashlib +import json +from dataclasses import dataclass +from datetime import datetime, timezone +from enum import Enum +from typing import Callable, Dict, Iterable, Mapping, MutableMapping, Sequence + +from library_sync import MatchResult, MatchStatus +from library_sync_review_state import ReviewStateStore, quality_delta + +DEFAULT_REPORT_VERSION = 1 + + +class ReportFormat(str, Enum): + """Supported export formats.""" + + JSON = "json" + CSV = "csv" + + @classmethod + def from_value(cls, value: str) -> "ReportFormat": + try: + return cls(value.lower()) + except Exception as exc: + raise ValueError(f"Unsupported report format: {value}") from exc + + +REQUIRED_FIELDS = [ + "schema_version", + "incoming_path", + "existing_path", + "status", + "distance", + "threshold_used", + "incoming_quality_score", + "existing_quality_score", + "quality_delta", + "user_flag", + "notes", + "timestamp", + "scan_config_hash", +] + +PreviewHandler = Callable[[Dict[str, int]], bool] + + +@dataclass +class ExportRecord: + """Row captured in the export plan/report.""" + + schema_version: int + incoming_path: str + existing_path: str | None + status: str + distance: float | None + threshold_used: float + incoming_quality_score: int + existing_quality_score: int | None + quality_delta: int | None + user_flag: str | None + notes: str + timestamp: str + scan_config_hash: str + + def to_dict(self) -> Dict[str, object]: + return { + "schema_version": self.schema_version, + "incoming_path": self.incoming_path, + "existing_path": self.existing_path, + "status": self.status, + "distance": self.distance, + "threshold_used": self.threshold_used, + "incoming_quality_score": self.incoming_quality_score, + "existing_quality_score": self.existing_quality_score, + "quality_delta": self.quality_delta, + "user_flag": self.user_flag, + "notes": self.notes, + "timestamp": self.timestamp, + "scan_config_hash": self.scan_config_hash, + } + + +def _normalize_scan_config(scan_config: Mapping[str, object] | object) -> Dict[str, object]: + if hasattr(scan_config, "to_dict"): + cfg = getattr(scan_config, "to_dict")() + if isinstance(cfg, Mapping): + return dict(cfg) + if isinstance(scan_config, Mapping): + return dict(scan_config) + raise TypeError("scan_config must be a mapping or expose a to_dict method.") + + +def calculate_scan_config_hash(scan_config: Mapping[str, object]) -> str: + """Return a deterministic hash for the current scan configuration.""" + canonical = json.dumps(scan_config, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def summarize_match_results(results: Iterable[MatchResult], flags: ReviewStateStore) -> Dict[str, int]: + """Summarize review outcomes for preview prior to export.""" + new_count = 0 + collision_count = 0 + low_conf_count = 0 + flagged_copy = 0 + flagged_replace = 0 + + for res in results: + if res.status == MatchStatus.NEW: + new_count += 1 + elif res.status in (MatchStatus.COLLISION, MatchStatus.EXACT_MATCH): + collision_count += 1 + elif res.status == MatchStatus.LOW_CONFIDENCE: + low_conf_count += 1 + + if flags.is_copy_flagged(res.incoming.track_id): + flagged_copy += 1 + if flags.replace_target(res.incoming.track_id): + flagged_replace += 1 + + return { + "new": new_count, + "collisions": collision_count, + "low_confidence": low_conf_count, + "flagged_copy": flagged_copy, + "flagged_replace": flagged_replace, + } + + +def build_export_records( + results: Sequence[MatchResult], + flags: ReviewStateStore, + scan_config: Mapping[str, object], + report_version: int = DEFAULT_REPORT_VERSION, + notes: MutableMapping[str, str] | None = None, + timestamp: datetime | None = None, +) -> tuple[list[ExportRecord], Dict[str, str]]: + """Create structured export rows without touching the filesystem.""" + normalized_cfg = _normalize_scan_config(scan_config) + scan_hash = calculate_scan_config_hash(normalized_cfg) + ts = timestamp or datetime.now(timezone.utc) + ts_str = ts.isoformat() + note_map = notes if notes is not None else flags.flags.notes + + rows: list[ExportRecord] = [] + for res in results: + incoming_id = res.incoming.track_id + user_flag: str | None = None + if flags.replace_target(incoming_id): + user_flag = "replace" + elif flags.is_copy_flagged(incoming_id): + user_flag = "copy" + + row = ExportRecord( + schema_version=report_version, + incoming_path=res.incoming.path, + existing_path=res.existing.path if res.existing else None, + status=res.status.value, + distance=res.distance, + threshold_used=res.threshold_used, + incoming_quality_score=res.incoming_score, + existing_quality_score=res.existing_score, + quality_delta=quality_delta(res), + user_flag=user_flag, + notes=note_map.get(incoming_id, "") if hasattr(note_map, "get") else "", + timestamp=ts_str, + scan_config_hash=scan_hash, + ) + rows.append(row) + + metadata = {"timestamp": ts_str, "scan_config_hash": scan_hash} + return rows, metadata + + +def export_report( + output_path: str, + results: Sequence[MatchResult], + flags: ReviewStateStore, + scan_config: Mapping[str, object], + fmt: str = ReportFormat.JSON.value, + report_version: int = DEFAULT_REPORT_VERSION, + notes: MutableMapping[str, str] | None = None, + timestamp: datetime | None = None, + preview_handler: PreviewHandler | None = None, +) -> str: + """Write a CSV or JSON report of the review state. + + The export routine deliberately avoids any move/replace logic and only + writes the requested report file after the caller has previewed counts. + """ + summary = summarize_match_results(results, flags) + if preview_handler: + proceed = preview_handler(summary) + if not proceed: + return "" + + records, metadata = build_export_records( + results, + flags, + scan_config, + report_version=report_version, + notes=notes, + timestamp=timestamp, + ) + format_enum = ReportFormat.from_value(fmt) + + if format_enum is ReportFormat.JSON: + payload = { + "schema_version": report_version, + "generated_at": metadata["timestamp"], + "scan_config_hash": metadata["scan_config_hash"], + "items": [rec.to_dict() for rec in records], + } + with open(output_path, "w", encoding="utf-8") as f: + json.dump(payload, f, indent=2) + else: + with open(output_path, "w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=REQUIRED_FIELDS) + writer.writeheader() + for rec in records: + writer.writerow(rec.to_dict()) + + return output_path diff --git a/library_sync_review_state.py b/library_sync_review_state.py index 73f47da..dfd655f 100644 --- a/library_sync_review_state.py +++ b/library_sync_review_state.py @@ -13,6 +13,7 @@ class ReviewFlags: copy: set[str] = field(default_factory=set) replace: Dict[str, str] = field(default_factory=dict) + notes: Dict[str, str] = field(default_factory=dict) class ReviewStateStore: @@ -39,6 +40,17 @@ def flag_for_replace(self, match: MatchResult) -> None: raise ValueError("Cannot flag for replace without a best match.") self.flags.replace[match.incoming.track_id] = match.existing.track_id + def set_note(self, incoming_track_id: str, note: str | None) -> None: + """Attach or clear a freeform note for an incoming row.""" + if note is None or not str(note).strip(): + self.flags.notes.pop(incoming_track_id, None) + return + self.flags.notes[incoming_track_id] = str(note) + + def note_for(self, incoming_track_id: str) -> str | None: + """Return a stored note for the incoming row, if present.""" + return self.flags.notes.get(incoming_track_id) + def replace_target(self, incoming_track_id: str) -> str | None: """Return the flagged replacement target, if any.""" return self.flags.replace.get(incoming_track_id) @@ -47,6 +59,7 @@ def clear_all(self) -> None: """Remove all flags across the review session.""" self.flags.copy.clear() self.flags.replace.clear() + self.flags.notes.clear() def reconcile_best_matches(self, results: Iterable[MatchResult]) -> List[str]: """Rebind replacement flags when the best match changes. diff --git a/tests/test_library_sync_review_report.py b/tests/test_library_sync_review_report.py new file mode 100644 index 0000000..257db21 --- /dev/null +++ b/tests/test_library_sync_review_report.py @@ -0,0 +1,178 @@ +import csv +import json +import sys +import types +from datetime import datetime, timezone + +# Stub heavy dependencies before importing modules under test +acoustid_stub = types.ModuleType("acoustid") +acoustid_stub.fingerprint_file = lambda path: (0, "stub-fp") +sys.modules.setdefault("acoustid", acoustid_stub) + +silence_stub = types.ModuleType("pydub.silence") +silence_stub.detect_nonsilent = lambda *args, **kwargs: [] + +audio_segment = type("AudioSegment", (), {"from_file": staticmethod(lambda path: None)}) +pydub_stub = types.ModuleType("pydub") +pydub_stub.AudioSegment = audio_segment +pydub_stub.silence = silence_stub + +sys.modules.setdefault("pydub", pydub_stub) +sys.modules.setdefault("pydub.silence", silence_stub) + +from library_sync import MatchResult, MatchStatus, TrackRecord +from library_sync_review_report import ( + DEFAULT_REPORT_VERSION, + calculate_scan_config_hash, + export_report, +) +from library_sync_review_state import ReviewStateStore + + +def _track(track_id: str, path: str, ext: str = ".mp3", bitrate: int = 192) -> TrackRecord: + return TrackRecord( + track_id=track_id, + path=path, + normalized_path=path, + ext=ext, + bitrate=bitrate, + size=1, + mtime=0.0, + fingerprint="fp", + tags={}, + duration=60, + ) + + +def _match( + incoming_id: str, + existing_id: str | None, + status: MatchStatus = MatchStatus.COLLISION, + incoming_score: int = 10, + existing_score: int | None = 5, +) -> MatchResult: + incoming = _track(incoming_id, f"/incoming/{incoming_id}") + existing = _track(existing_id, f"/library/{existing_id}") if existing_id else None + return MatchResult( + incoming=incoming, + existing=existing, + status=status, + distance=0.1 if existing else None, + threshold_used=0.3, + near_miss_margin=0.05, + confidence=0.8 if existing else 0.0, + candidates=[], + quality_label="Potential Upgrade" + if existing_score is not None and incoming_score > existing_score + else "Keep Existing", + incoming_score=incoming_score, + existing_score=existing_score, + ) + + +def test_export_report_json_includes_required_fields(tmp_path) -> None: + store = ReviewStateStore() + results = [ + _match("inc-new", None, status=MatchStatus.NEW, incoming_score=7, existing_score=None), + _match("inc-collision", "lib-b", status=MatchStatus.COLLISION, incoming_score=9, existing_score=5), + ] + store.flag_for_copy(results[0].incoming.track_id) + store.flag_for_replace(results[1]) + store.set_note(results[0].incoming.track_id, "verify tags") + + scan_config = {"global_threshold": 0.3, "per_format_overrides": {"flac": 0.2}} + ts = datetime(2024, 1, 2, tzinfo=timezone.utc) + captured_summary: dict[str, int] = {} + + def preview(summary: dict[str, int]) -> bool: + captured_summary.update(summary) + return True + + path = tmp_path / "report.json" + export_report( + str(path), + results, + store, + scan_config, + fmt="json", + report_version=2, + timestamp=ts, + preview_handler=preview, + ) + + data = json.loads(path.read_text()) + assert data["schema_version"] == 2 + assert data["scan_config_hash"] == calculate_scan_config_hash(scan_config) + assert len(data["items"]) == 2 + + first = data["items"][0] + assert first["incoming_path"].endswith("inc-new") + assert first["existing_path"] is None + assert first["user_flag"] == "copy" + assert first["notes"] == "verify tags" + assert first["timestamp"] == ts.isoformat() + assert captured_summary["flagged_copy"] == 1 + assert captured_summary["flagged_replace"] == 1 + + +def test_export_report_csv_and_summary_counts(tmp_path) -> None: + store = ReviewStateStore() + results = [ + _match("inc-new", None, status=MatchStatus.NEW, incoming_score=3, existing_score=None), + _match("inc-collision", "lib-1", status=MatchStatus.COLLISION, incoming_score=5, existing_score=2), + _match( + "inc-low", + "lib-low", + status=MatchStatus.LOW_CONFIDENCE, + incoming_score=4, + existing_score=6, + ), + ] + store.flag_for_copy(results[2].incoming.track_id) + store.flag_for_replace(results[1]) + + scan_config = {"default": 0.3} + ts = datetime(2024, 1, 3, tzinfo=timezone.utc) + previews: list[dict[str, int]] = [] + + def preview(summary: dict[str, int]) -> bool: + previews.append(summary) + return True + + path = tmp_path / "report.csv" + export_report( + str(path), + results, + store, + scan_config, + fmt="csv", + timestamp=ts, + preview_handler=preview, + ) + + rows = list(csv.DictReader(path.read_text().splitlines())) + assert len(rows) == len(results) + assert rows[0]["schema_version"] == str(DEFAULT_REPORT_VERSION) + assert rows[1]["existing_path"].endswith("lib-1") + assert rows[2]["user_flag"] == "copy" + assert rows[0]["status"] == MatchStatus.NEW.value + assert previews[0]["collisions"] == 1 + assert previews[0]["low_confidence"] == 1 + assert previews[0]["flagged_replace"] == 1 + + +def test_export_report_aborts_when_preview_rejected(tmp_path) -> None: + store = ReviewStateStore() + results = [_match("inc-new", None, status=MatchStatus.NEW, existing_score=None)] + path = tmp_path / "rejected.json" + + def reject(summary: dict[str, int]) -> bool: + return False + + before = set(tmp_path.iterdir()) + returned_path = export_report(str(path), results, store, {}, preview_handler=reject) + after = set(tmp_path.iterdir()) + + assert returned_path == "" + assert not path.exists() + assert before == after From b2d8d9df2547139b16847a9e4335596175473ded Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sat, 20 Dec 2025 11:25:09 -0500 Subject: [PATCH 128/606] Implement review sync UI behaviors --- fingerprint_generator.py | 32 +- library_sync.py | 51 ++- library_sync_review.py | 484 ++++++++++++++++++++++++++-- tests/test_fingerprint_generator.py | 22 ++ tests/test_library_sync.py | 8 +- 5 files changed, 564 insertions(+), 33 deletions(-) diff --git a/fingerprint_generator.py b/fingerprint_generator.py index 3844bed..fb5d5ba 100644 --- a/fingerprint_generator.py +++ b/fingerprint_generator.py @@ -1,7 +1,8 @@ import os import sqlite3 -from typing import Callable, Iterable, List, Tuple import tempfile +import threading +from typing import Callable, Iterable, List, Tuple from pydub import AudioSegment, silence @@ -97,6 +98,7 @@ def compute_fingerprints_parallel( max_workers: int | None = None, trim_silence: bool = False, phase: str = "A", + cancel_event: threading.Event | None = None, ) -> list[tuple[str, int | None, str | None]]: """Compute fingerprints and return ``(path, duration, fingerprint)`` results. @@ -111,6 +113,9 @@ def compute_fingerprints_parallel( def progress_callback(current: int, total: int, msg: str, _phase: str) -> None: pass + if cancel_event is None: + cancel_event = threading.Event() + if max_workers is None: max_workers = os.cpu_count() or 1 @@ -139,6 +144,9 @@ def progress_callback(current: int, total: int, msg: str, _phase: str) -> None: completed = 0 for path in audio_files: + if cancel_event.is_set(): + log_callback("Cancellation requested before fingerprint scheduling") + break mtime = os.path.getmtime(ensure_long_path(path)) cache_entry = cached.get(path) if cache_entry and cache_entry[2] is not None and cache_entry[0] == mtime: @@ -153,13 +161,23 @@ def progress_callback(current: int, total: int, msg: str, _phase: str) -> None: if pending: with ProcessPoolExecutor(max_workers=max_workers) as exe: - for path, duration, fp_hash, err in exe.map( - compute_fingerprint_for_file, work_items - ): + futures = [exe.submit(compute_fingerprint_for_file, item) for item in work_items] + for fut in futures: + if cancel_event.is_set(): + cancelled = 0 + for remaining in futures: + if not remaining.done() and remaining.cancel(): + cancelled += 1 + log_callback(f"Cancellation requested; attempting to stop after current file ({cancelled} cancelled)") + break + + path, duration, fp_hash, err = fut.result() completed += 1 if err: log_callback(f" ! Failed fingerprint {path}: {err}") progress_callback(completed, total, path, phase) + if cancel_event.is_set(): + break continue mtime = os.path.getmtime(ensure_long_path(path)) conn.execute( @@ -171,6 +189,9 @@ def progress_callback(current: int, total: int, msg: str, _phase: str) -> None: progress_callback(completed, total, path, phase) if completed % 50 == 0 or completed == total: log_callback(f" • Fingerprinting {completed}/{total}") + if cancel_event.is_set(): + log_callback("Cancellation requested; stopping after current fingerprint") + break else: log_callback("All fingerprints loaded from cache") @@ -188,6 +209,7 @@ def compute_fingerprints( max_workers: int | None = None, trim_silence: bool = False, phase: str = "A", + cancel_event: threading.Event | None = None, ) -> None: """Backward-compatible wrapper around :func:`compute_fingerprints_parallel`.""" return compute_fingerprints_parallel( @@ -198,5 +220,5 @@ def compute_fingerprints( max_workers, trim_silence, phase, + cancel_event, ) - diff --git a/library_sync.py b/library_sync.py index 0e9f173..589278f 100644 --- a/library_sync.py +++ b/library_sync.py @@ -2,8 +2,10 @@ from __future__ import annotations import hashlib +import json import logging import os +import threading from dataclasses import dataclass, field from enum import Enum from typing import Callable, Dict, Iterable, List, Optional, Tuple @@ -199,13 +201,19 @@ def _scan_folder( on_record: Callable[[TrackRecord], None] | None = None, log_callback: Callable[[str], None] | None = None, progress_callback: Callable[[int, int, str, str], None] | None = None, + cancel_event: threading.Event | None = None, ) -> List[TrackRecord]: """Enumerate audio files, normalize paths, and attach cached fingerprints.""" + cancel_event = cancel_event or threading.Event() record_event(f"library_sync: scanning folder {folder}") _dlog(f"DEBUG: Scanning folder {folder}", log_callback) entries: List[Dict[str, object]] = [] total_files = 0 for dirpath, _dirs, files in os.walk(folder): + if cancel_event.is_set(): + if log_callback: + log_callback(json.dumps({"event": "scan_cancelled", "folder": folder, "stage": "walk"})) + break for fname in files: ext = os.path.splitext(fname)[1].lower() if ext not in idx.SUPPORTED_EXTS: @@ -218,6 +226,8 @@ def _scan_folder( size = stat.st_size except OSError as e: _dlog(f"DEBUG: Skipping unreadable file {path}: {e}", log_callback) + if log_callback: + log_callback(json.dumps({"event": "skip", "path": path, "reason": str(e)})) continue tags = idx.get_tags(path) bitrate = 0 @@ -260,6 +270,8 @@ def _scan_folder( entry["duration"] = row[2] entry["fingerprint"] = row[3] _dlog(f"DEBUG: cache hit {entry['path']}", log_callback) + if log_callback: + log_callback(json.dumps({"event": "cache_hit", "path": entry["path"]})) else: if row: msg = ( @@ -269,16 +281,31 @@ def _scan_folder( ) if log_callback: log_callback(msg) + log_callback(json.dumps({"event": "cache_miss", "path": entry["path"], "reason": "stale"})) _dlog(msg, log_callback) to_compute.append(entry["path"]) def fp_progress(current: int, total: int, path: str, phase: str) -> None: if progress_callback: progress_callback(current, total, path, phase) + if log_callback and phase.startswith("fp") and current and total: + log_callback( + json.dumps( + { + "event": "fingerprint_progress", + "path": path, + "current": current, + "total": total, + "phase": phase, + } + ) + ) if to_compute: + if log_callback: + log_callback(json.dumps({"event": "fingerprint_start", "pending": len(to_compute), "folder": folder})) for result in fingerprint_generator.compute_fingerprints_parallel( - to_compute, db_path, log_callback, fp_progress + to_compute, db_path, log_callback, fp_progress, cancel_event=cancel_event ): path = result[0] duration = result[1] if len(result) > 2 else None @@ -292,6 +319,9 @@ def fp_progress(current: int, total: int, path: str, phase: str) -> None: "INSERT OR REPLACE INTO fingerprints (path, mtime, size, duration, fingerprint) VALUES (?, ?, ?, ?, ?)", (path, entry["mtime"], entry["size"], duration, fp), ) + if cancel_event.is_set(): + _dlog(f"DEBUG: cancellation after fingerprinting {path}", log_callback) + break conn.commit() conn.close() @@ -427,6 +457,7 @@ def compare_libraries( thresholds: Dict[str, float] | None = None, log_callback: Callable[[str], None] | None = None, progress_callback: Callable[[int, int, str, str], None] | None = None, + cancel_event: threading.Event | None = None, ) -> Dict[str, object]: """Return classification of incoming files vs. an existing library. @@ -442,6 +473,7 @@ def compare_libraries( record_event( f"library_sync: comparing {library_folder} to {incoming_folder}" ) + cancel_event = cancel_event or threading.Event() if thresholds is None: if threshold is not None: thresholds = {"default": threshold} @@ -449,8 +481,20 @@ def compare_libraries( thresholds = DEFAULT_FP_THRESHOLDS fmt_priority = fmt_priority or FORMAT_PRIORITY - lib_records = _scan_folder(library_folder, db_path, log_callback=log_callback, progress_callback=progress_callback) - inc_records = _scan_folder(incoming_folder, db_path, log_callback=log_callback, progress_callback=progress_callback) + lib_records = _scan_folder( + library_folder, + db_path, + log_callback=log_callback, + progress_callback=progress_callback, + cancel_event=cancel_event, + ) + inc_records = _scan_folder( + incoming_folder, + db_path, + log_callback=log_callback, + progress_callback=progress_callback, + cancel_event=cancel_event, + ) match_results = _match_tracks(inc_records, lib_records, thresholds, fmt_priority, log_callback=log_callback) @@ -479,6 +523,7 @@ def compare_libraries( "library_records": [r.to_dict() for r in lib_records], "incoming_records": [r.to_dict() for r in inc_records], "matches": [m.to_dict() for m in match_results], + "partial": cancel_event.is_set(), } record_event( "library_sync: comparison complete " diff --git a/library_sync_review.py b/library_sync_review.py index 5dc8ffd..a253040 100644 --- a/library_sync_review.py +++ b/library_sync_review.py @@ -1,15 +1,22 @@ """Review-first Library Sync entry point and session models.""" from __future__ import annotations +import json +import os +import threading from dataclasses import dataclass, field +from datetime import datetime from enum import Enum -from typing import Any, Dict +from typing import Any, Dict, Iterable, List, Sequence import tkinter as tk from tkinter import filedialog, messagebox, ttk -from config import DEFAULT_FP_THRESHOLDS, load_config, save_config +from config import DEFAULT_FP_THRESHOLDS, FORMAT_PRIORITY, load_config, save_config +import library_sync +from library_sync import MatchResult, MatchStatus, TrackRecord from library_sync_review_report import DEFAULT_REPORT_VERSION +from library_sync_review_state import ReviewStateStore REVIEW_CONFIG_KEY = "library_sync_review" REVIEW_FLAG_KEY = "use_library_sync_review" @@ -179,7 +186,7 @@ class LibrarySyncReviewWindow(tk.Toplevel): def __init__(self, master: tk.Widget | None = None): super().__init__(master) self.title("Library Sync (Review)") - self.resizable(True, False) + self.resizable(True, True) self.session = load_scan_session() self.library_var = tk.StringVar(value=self.session.library_root) @@ -189,15 +196,53 @@ def __init__(self, master: tk.Widget | None = None): self.report_version_var = tk.StringVar(value=str(self.session.exported_report_version)) self.scan_state_var = tk.StringVar(value=self._describe_state(self.session.scan_state)) + # Scan + progress state + self.library_cancel = threading.Event() + self.incoming_cancel = threading.Event() + self.progress_vars: dict[str, tk.DoubleVar] = { + "library": tk.DoubleVar(value=0), + "incoming": tk.DoubleVar(value=0), + } + self.progress_totals: dict[str, int] = {"library": 0, "incoming": 0} + self.progress_labels: dict[str, tk.StringVar] = { + "library": tk.StringVar(value="Idle"), + "incoming": tk.StringVar(value="Idle"), + } + self.partial_labels: dict[str, tk.StringVar] = { + "library": tk.StringVar(value=""), + "incoming": tk.StringVar(value=""), + } + self.phase_labels: dict[str, tk.StringVar] = { + "library": tk.StringVar(value=""), + "incoming": tk.StringVar(value=""), + } + self.partial_flags: dict[str, bool] = {"library": False, "incoming": False} + + # Review + matching state + self.state_store = ReviewStateStore() + self.library_records: list[TrackRecord] = [] + self.incoming_records: list[TrackRecord] = [] + self.match_results: list[MatchResult] = [] + self.match_by_incoming: dict[str, MatchResult] = {} + self.match_by_existing: dict[str, list[MatchResult]] = {} + self.library_index: dict[str, TrackRecord] = {} + self.incoming_index: dict[str, TrackRecord] = {} + + # Logs + self.log_entries: list[dict[str, object]] = [] + self.LOG_LIMIT = 400 + self._build_ui() self.protocol("WM_DELETE_WINDOW", self._on_close) def _build_ui(self) -> None: padding = {"padx": 10, "pady": 5} - folder_frame = ttk.LabelFrame(self, text="Folders") - folder_frame.pack(fill="x", **padding) + root = ttk.Frame(self) + root.pack(fill="both", expand=True) + folder_frame = ttk.LabelFrame(root, text="Folders") + folder_frame.pack(fill="x", **padding) self._add_browse_row( folder_frame, "Existing Library:", self.library_var, self._browse_library ) @@ -205,7 +250,7 @@ def _build_ui(self) -> None: folder_frame, "Incoming Folder:", self.incoming_var, self._browse_incoming ) - scan_frame = ttk.LabelFrame(self, text="Scan Configuration") + scan_frame = ttk.LabelFrame(root, text="Scan Configuration") scan_frame.pack(fill="x", **padding) thr_row = ttk.Frame(scan_frame) @@ -232,26 +277,54 @@ def _build_ui(self) -> None: self.overrides_text.pack(fill="x", expand=True, pady=(2, 0)) self.overrides_text.insert("1.0", self._format_overrides()) - meta_frame = ttk.LabelFrame(self, text="Session") - meta_frame.pack(fill="x", **padding) - meta_row = ttk.Frame(meta_frame) - meta_row.pack(fill="x", padx=5, pady=(5, 2)) + meta_row = ttk.Frame(scan_frame) + meta_row.pack(fill="x", padx=5, pady=(2, 5)) + ttk.Label(meta_row, text="Report Version:").pack(side="left") + ttk.Entry(meta_row, textvariable=self.report_version_var, width=6).pack(side="left", padx=(5, 10)) ttk.Label(meta_row, text="Scan State:").pack(side="left") ttk.Label(meta_row, textvariable=self.scan_state_var).pack(side="left", padx=(5, 0)) - version_row = ttk.Frame(meta_frame) - version_row.pack(fill="x", padx=5, pady=(2, 5)) - ttk.Label(version_row, text="Exported Report Version:").pack(side="left") - ttk.Entry(version_row, textvariable=self.report_version_var, width=6).pack( + controls = ttk.LabelFrame(root, text="Scan Progress") + controls.pack(fill="x", **padding) + controls.columnconfigure((0, 1), weight=1) + self._build_scan_column(controls, "Existing Library", "library", 0) + self._build_scan_column(controls, "Incoming Folder", "incoming", 1) + + body = ttk.Frame(root) + body.pack(fill="both", expand=True, **padding) + body.columnconfigure(0, weight=1) + body.columnconfigure(1, weight=1) + body.rowconfigure(0, weight=2) + body.rowconfigure(1, weight=1) + + # Two-list UI + incoming_frame = ttk.LabelFrame(body, text="Incoming Tracks") + existing_frame = ttk.LabelFrame(body, text="Existing Tracks") + incoming_frame.grid(row=0, column=0, sticky="nsew", padx=(0, 5)) + existing_frame.grid(row=0, column=1, sticky="nsew", padx=(5, 0)) + self._build_incoming_tree(incoming_frame) + self._build_existing_tree(existing_frame) + + # Inspector drawer + inspector = ttk.LabelFrame(body, text="Inspector") + inspector.grid(row=1, column=0, columnspan=2, sticky="nsew", pady=(8, 0)) + inspector.columnconfigure((0, 1), weight=1) + self.inspector_text = tk.Text(inspector, height=8, state="disabled", wrap="word") + self.inspector_text.grid(row=0, column=0, columnspan=2, sticky="nsew", padx=5, pady=5) + + # Log panel + log_frame = ttk.LabelFrame(root, text="Logs") + log_frame.pack(fill="both", expand=True, **padding) + log_frame.columnconfigure(0, weight=1) + self.log_widget = tk.Text(log_frame, height=8, state="disabled", wrap="word") + self.log_widget.grid(row=0, column=0, sticky="nsew", padx=5, pady=5) + log_btns = ttk.Frame(log_frame) + log_btns.grid(row=1, column=0, sticky="ew", padx=5, pady=(0, 5)) + ttk.Button(log_btns, text="Export Logs…", command=self._export_logs).pack(side="left") + ttk.Button(log_btns, text="Save Session", command=self._persist_session).pack( side="left", padx=(5, 0) ) - - btn_row = ttk.Frame(self) - btn_row.pack(fill="x", pady=(0, 10)) - ttk.Button(btn_row, text="Save Session", command=self._persist_session).pack( - side="left", padx=(10, 5) - ) - ttk.Button(btn_row, text="Close", command=self._on_close).pack(side="right", padx=(5, 10)) + ttk.Button(log_btns, text="Close", command=self._on_close).pack(side="right") def _add_browse_row( self, @@ -266,6 +339,53 @@ def _add_browse_row( ttk.Entry(row, textvariable=variable).pack(side="left", fill="x", expand=True, padx=(5, 5)) ttk.Button(row, text="Browse…", command=command).pack(side="left") + def _build_scan_column(self, parent: ttk.Frame, title: str, kind: str, column: int) -> None: + col = ttk.Frame(parent) + col.grid(row=0, column=column, sticky="nsew", padx=5, pady=5) + ttk.Label(col, text=title).pack(anchor="w") + ttk.Label(col, textvariable=self.progress_labels[kind]).pack(anchor="w") + ttk.Progressbar(col, variable=self.progress_vars[kind], maximum=1).pack( + fill="x", pady=2 + ) + ttk.Label(col, textvariable=self.phase_labels[kind], foreground="#555").pack(anchor="w") + ttk.Label(col, textvariable=self.partial_labels[kind], foreground="#a64e00").pack(anchor="w") + btns = ttk.Frame(col) + btns.pack(fill="x", pady=(4, 0)) + ttk.Button(btns, text="Scan", command=lambda k=kind: self._start_scan(k)).pack( + side="left" + ) + ttk.Button( + btns, + text="Cancel", + command=lambda k=kind: self._cancel_scan(k), + ).pack(side="left", padx=(5, 0)) + + def _build_incoming_tree(self, parent: ttk.Frame) -> None: + columns = ("name", "chips", "distance") + tree = ttk.Treeview(parent, columns=columns, show="headings", selectmode="browse") + tree.heading("name", text="Track") + tree.heading("chips", text="Chips") + tree.heading("distance", text="Distance") + tree.column("name", width=200, anchor="w") + tree.column("chips", width=200, anchor="w") + tree.column("distance", width=80, anchor="center") + tree.pack(fill="both", expand=True, padx=5, pady=5) + tree.bind("<>", self._on_incoming_select) + self.incoming_tree = tree + + def _build_existing_tree(self, parent: ttk.Frame) -> None: + columns = ("name", "chips", "incoming_count") + tree = ttk.Treeview(parent, columns=columns, show="headings", selectmode="browse") + tree.heading("name", text="Track") + tree.heading("chips", text="Chips") + tree.heading("incoming_count", text="Best Matches") + tree.column("name", width=200, anchor="w") + tree.column("chips", width=200, anchor="w") + tree.column("incoming_count", width=100, anchor="center") + tree.pack(fill="both", expand=True, padx=5, pady=5) + tree.bind("<>", self._on_existing_select) + self.existing_tree = tree + def _browse_library(self) -> None: folder = filedialog.askdirectory(title="Select Library Root", initialdir=self.library_var.get() or None) if folder: @@ -327,6 +447,16 @@ def _persist_session(self) -> bool: ) save_scan_session(self.session) self.scan_state_var.set(self._describe_state(self.session.scan_state)) + self._log_event( + "config_change", + "Updated scan configuration", + { + "global_threshold": global_thr, + "overrides": overrides, + "preset": self.preset_var.get().strip() or None, + "report_version": report_ver, + }, + ) return True def _describe_state(self, state: ScanState) -> str: @@ -339,3 +469,315 @@ def preview_report(self, summary: dict[str, int]) -> bool: def _on_close(self) -> None: if self._persist_session(): self.destroy() + + # ── Scan + Progress ------------------------------------------------- + def _cancel_event_for(self, kind: str) -> threading.Event: + return self.library_cancel if kind == "library" else self.incoming_cancel + + def _start_scan(self, kind: str) -> None: + if not self._persist_session(): + return + folder = self.library_var.get().strip() if kind == "library" else self.incoming_var.get().strip() + if not folder: + messagebox.showerror("Missing Folder", f"Please choose a {kind} folder before scanning.") + return + cancel_event = self._cancel_event_for(kind) + cancel_event.clear() + self.progress_vars[kind].set(0) + self.progress_totals[kind] = 0 + self.progress_labels[kind].set("Scanning…") + self.phase_labels[kind].set("") + self.partial_labels[kind].set("") + self.partial_flags[kind] = False + self.session.scan_state = ScanState.SCANNING_LIBRARY if kind == "library" else ScanState.SCANNING_INCOMING + self.scan_state_var.set(self._describe_state(self.session.scan_state)) + self._log_event("scan_start", f"Scanning {folder}", {"kind": kind, "folder": folder}) + thread = threading.Thread(target=self._scan_worker, args=(kind, folder, cancel_event), daemon=True) + thread.start() + + def _cancel_scan(self, kind: str) -> None: + cancel_event = self._cancel_event_for(kind) + cancel_event.set() + self.partial_labels[kind].set("Cancelling… Partial results will be kept.") + self._log_event("scan_cancel", f"Cancellation requested for {kind}", {"kind": kind}) + + def _scan_worker(self, kind: str, folder: str, cancel_event: threading.Event) -> None: + db_root = self.library_var.get().strip() or folder + docs_dir = os.path.join(db_root, "Docs") + os.makedirs(docs_dir, exist_ok=True) + db_path = os.path.join(docs_dir, ".soundvault.db") + + progress_cb = self._make_progress_handler(kind) + try: + records = library_sync._scan_folder( + folder, + db_path, + log_callback=lambda msg: self._log_scan_message(kind, msg), + progress_callback=progress_cb, + cancel_event=cancel_event, + ) + except Exception as exc: # pragma: no cover - UI only + self._log_event("scan_error", str(exc), {"kind": kind}) + self.after(0, lambda e=exc: messagebox.showerror("Scan Failed", str(e))) + return + + self.after(0, lambda recs=records: self._on_scan_complete(kind, recs)) + + def _make_progress_handler(self, kind: str): + def handler(current: int, total: int, path: str, phase: str) -> None: + self.after(0, lambda c=current, t=total, p=path, ph=phase: self._update_progress(kind, c, t, p, ph)) + + return handler + + def _update_progress(self, kind: str, current: int, total: int, path: str, phase: str) -> None: + if total: + self.progress_totals[kind] = total + self.progress_vars[kind].set(current / total if total else 0) + label = f"{phase.title()}: {os.path.basename(path) if path else ''}".strip() + self.phase_labels[kind].set(label) + + def _on_scan_complete(self, kind: str, records: Sequence[TrackRecord]) -> None: + cancel_event = self._cancel_event_for(kind) + partial = cancel_event.is_set() + self.partial_flags[kind] = partial + status = "Partial" if partial else "Complete" + self.progress_labels[kind].set(status) + self.partial_labels[kind].set("Partial" if partial else "") + self.phase_labels[kind].set("") + if kind == "library": + self.library_records = list(records) + self.library_index = {r.track_id: r for r in records} + else: + self.incoming_records = list(records) + self.incoming_index = {r.track_id: r for r in records} + + self._log_event( + "scan_complete", + f"Finished {kind} scan ({status})", + {"kind": kind, "count": len(records), "partial": partial}, + ) + if partial: + self.session.scan_state = ScanState.CANCELLED + elif self.library_records and self.incoming_records: + self.session.scan_state = ScanState.COMPLETE + else: + self.session.scan_state = ScanState.READY + self.scan_state_var.set(self._describe_state(self.session.scan_state)) + save_scan_session(self.session) + self._maybe_match() + + # ── Matching + Display ---------------------------------------------- + def _maybe_match(self) -> None: + if not self.library_records or not self.incoming_records: + return + try: + overrides = self._parse_overrides() or {} + global_thr = float(self.global_threshold_var.get()) + except Exception: + overrides = self._parse_overrides() or {} + global_thr = DEFAULT_FP_THRESHOLDS.get("default", 0.3) + thresholds = {"default": global_thr, **overrides} + self._log_event("match_start", "Computing matches", {"thresholds": thresholds}) + results = library_sync._match_tracks( + self.incoming_records, + self.library_records, + thresholds, + FORMAT_PRIORITY, + log_callback=lambda msg: self._log_event("match_log", str(msg)), + ) + self.match_results = results + self.match_by_incoming = {res.incoming.track_id: res for res in results} + match_by_existing: dict[str, list[MatchResult]] = {} + for res in results: + if res.existing: + match_by_existing.setdefault(res.existing.track_id, []).append(res) + self.match_by_existing = match_by_existing + self._log_event("match_stats", "Updated matching statistics", self._summarize_matches(results)) + self._render_lists() + self._update_inspector() + + def _render_lists(self) -> None: + for tree in (self.incoming_tree, self.existing_tree): + for iid in tree.get_children(): + tree.delete(iid) + + for res in self.match_results: + chips = ", ".join(self._chips_for_incoming(res)) + distance = "-" if res.distance is None else f"{res.distance:.3f}" + name = os.path.basename(res.incoming.path) + self.incoming_tree.insert( + "", + "end", + iid=res.incoming.track_id, + values=(name, chips, distance), + ) + + for rec in self.library_records: + linked = self.match_by_existing.get(rec.track_id, []) + chips = ", ".join(self._chips_for_existing(linked)) + name = os.path.basename(rec.path) + self.existing_tree.insert( + "", + "end", + iid=rec.track_id, + values=(name, chips, len(linked)), + ) + + def _chips_for_incoming(self, res: MatchResult) -> List[str]: + chips: List[str] = [] + status_map = { + MatchStatus.NEW: "New", + MatchStatus.COLLISION: "Collision", + MatchStatus.EXACT_MATCH: "Exact Match", + MatchStatus.LOW_CONFIDENCE: "Low Confidence", + } + chips.append(status_map.get(res.status, res.status.value.title())) + if res.quality_label: + chips.append(res.quality_label) + if not res.incoming.tags: + chips.append("Missing Metadata") + if self.partial_flags.get("incoming") or self.partial_flags.get("library"): + chips.append("Partial") + return chips + + def _chips_for_existing(self, linked_matches: Iterable[MatchResult]) -> List[str]: + chips: List[str] = [] + linked = list(linked_matches) + if linked: + chips.append("Best Match") + # surface aggregate quality direction + if any(res.quality_label == "Potential Upgrade" for res in linked): + chips.append("Potential Upgrade") + elif any(res.quality_label == "Keep Existing" for res in linked): + chips.append("Keep Existing") + if self.partial_flags.get("library"): + chips.append("Partial") + return chips or ["Unmatched"] + + def _on_incoming_select(self, _event=None) -> None: + sel = self.incoming_tree.selection() + if not sel: + return + incoming_id = sel[0] + match = self.match_by_incoming.get(incoming_id) + if match and match.existing: + self.existing_tree.selection_set(match.existing.track_id) + self.existing_tree.see(match.existing.track_id) + self._update_inspector(incoming_id=incoming_id) + + def _on_existing_select(self, _event=None) -> None: + sel = self.existing_tree.selection() + if not sel: + return + existing_id = sel[0] + linked = self.match_by_existing.get(existing_id, []) + if linked: + target = linked[0].incoming.track_id + self.incoming_tree.selection_set(target) + self.incoming_tree.see(target) + self._update_inspector(incoming_id=target) + else: + self._update_inspector(existing_id=existing_id) + + def _update_inspector(self, incoming_id: str | None = None, existing_id: str | None = None) -> None: + res: MatchResult | None = None + if incoming_id: + res = self.match_by_incoming.get(incoming_id) + elif existing_id: + linked = self.match_by_existing.get(existing_id, []) + res = linked[0] if linked else None + + lines: List[str] = [] + if res: + inc = res.incoming + lines.append("Incoming Track") + lines.append(f" • Path: {inc.path}") + lines.append(f" • Status: {', '.join(self._chips_for_incoming(res))}") + lines.append(f" • Ext: {inc.ext} | Bitrate: {inc.bitrate or 'n/a'} | Duration: {inc.duration or 'n/a'}") + lines.append(f" • Fingerprint Distance: {res.distance if res.distance is not None else 'n/a'}") + lines.append(f" • Threshold Used: {res.threshold_used}") + lines.append(f" • Confidence: {res.confidence:.2f}") + if res.existing: + lines.append("") + lines.append("Best Match") + lines.append(f" • Path: {res.existing.path}") + lines.append(f" • Quality: {res.quality_label or 'n/a'}") + lines.append(f" • Match Distance: {res.distance if res.distance is not None else 'n/a'}") + else: + lines.append("") + lines.append("Best Match") + lines.append(" • None") + elif existing_id: + rec = self.library_index.get(existing_id) + if rec: + lines.append("Existing Track") + lines.append(f" • Path: {rec.path}") + lines.append(f" • Ext: {rec.ext} | Bitrate: {rec.bitrate or 'n/a'} | Duration: {rec.duration or 'n/a'}") + lines.append(" • No incoming matches yet.") + + self.inspector_text.configure(state="normal") + self.inspector_text.delete("1.0", "end") + self.inspector_text.insert("1.0", "\n".join(lines) if lines else "Select a row to inspect details.") + self.inspector_text.configure(state="disabled") + + def _summarize_matches(self, results: Iterable[MatchResult]) -> Dict[str, int]: + res_list = list(results) + counts = {"new": 0, "collision": 0, "exact": 0, "low_confidence": 0} + for res in res_list: + if res.status == MatchStatus.NEW: + counts["new"] += 1 + elif res.status == MatchStatus.COLLISION: + counts["collision"] += 1 + elif res.status == MatchStatus.EXACT_MATCH: + counts["exact"] += 1 + elif res.status == MatchStatus.LOW_CONFIDENCE: + counts["low_confidence"] += 1 + counts["total"] = len(res_list) + return counts + + # ── Logging --------------------------------------------------------- + def _log_event(self, event_type: str, message: str, data: Dict[str, object] | None = None) -> None: + entry = { + "timestamp": datetime.utcnow().isoformat(), + "type": event_type, + "message": message, + } + if data: + entry["data"] = data + self.log_entries.append(entry) + if len(self.log_entries) > self.LOG_LIMIT: + self.log_entries = self.log_entries[-self.LOG_LIMIT :] + self._append_log(entry) + + def _append_log(self, entry: Dict[str, object]) -> None: + self.log_widget.configure(state="normal") + line = f"[{entry['timestamp']}] {entry['type']}: {entry['message']}" + if "data" in entry: + line += f" {entry['data']}" + self.log_widget.insert("end", line + "\n") + self.log_widget.see("end") + self.log_widget.configure(state="disabled") + + def _export_logs(self) -> None: + path = filedialog.asksaveasfilename( + title="Export Logs", + defaultextension=".json", + filetypes=[("JSON", "*.json")], + ) + if not path: + return + with open(path, "w", encoding="utf-8") as f: + json.dump(self.log_entries, f, indent=2) + self._log_event("export_logs", f"Saved logs to {path}") + + def _log_scan_message(self, kind: str, msg: object) -> None: + if isinstance(msg, str): + try: + data = json.loads(msg) + self._log_event(f"{kind}_scan", data.get("event", "scan"), data) + except Exception: + self._log_event(f"{kind}_scan", msg) + elif isinstance(msg, dict): + self._log_event(f"{kind}_scan", msg.get("event", "scan"), msg) + else: + self._log_event(f"{kind}_scan", str(msg)) diff --git a/tests/test_fingerprint_generator.py b/tests/test_fingerprint_generator.py index bdef212..056f9fb 100644 --- a/tests/test_fingerprint_generator.py +++ b/tests/test_fingerprint_generator.py @@ -81,6 +81,28 @@ def __exit__(self, exc_type, exc, tb): def map(self, fn, iterable): return map(fn, iterable) + def submit(self, fn, arg): + class _Future: + def __init__(self, fn, arg): + self._fn = fn + self._arg = arg + self._result = None + self._executed = False + + def result(self): + if not self._executed: + self._result = self._fn(self._arg) + self._executed = True + return self._result + + def done(self): + return self._executed + + def cancel(self): + return False + + return _Future(fn, arg) + def test_mtime_change_triggers_recompute(monkeypatch, tmp_path): audio = tmp_path / "song.mp3" diff --git a/tests/test_library_sync.py b/tests/test_library_sync.py index 0f3b0f5..153e76a 100644 --- a/tests/test_library_sync.py +++ b/tests/test_library_sync.py @@ -27,7 +27,7 @@ def File(path, easy=False): fingerprint_stub = types.ModuleType('fingerprint_generator') -def fake_compute(paths, db_path, log_callback=None, progress_callback=None): +def fake_compute(paths, db_path, log_callback=None, progress_callback=None, cancel_event=None): for p in paths: yield p, 0, fp_map.get(os.path.basename(p), 'x') @@ -106,7 +106,7 @@ def fake_fp(path, **kw): return "1 1 1 1 1 1 1 2" return "1 1 1 1 1 1 1 1" - def fake_compute(paths, db_path, log_callback=None, progress_callback=None): + def fake_compute(paths, db_path, log_callback=None, progress_callback=None, cancel_event=None): for p in paths: fp = fake_fp(p) yield p, 0, fp @@ -177,7 +177,7 @@ def fake_fp(path, **kw): return "1 2 3 4" return "1 2 3 5" - def fake_compute(paths, db_path, log_callback=None, progress_callback=None): + def fake_compute(paths, db_path, log_callback=None, progress_callback=None, cancel_event=None): for p in paths: fp = fake_fp(p) yield p, 0, fp @@ -225,7 +225,7 @@ def fake_fp(path, **kw): return "10 20 30 40 50" return "10 20 30 40 51" - def fake_compute(paths, db_path, log_callback=None, progress_callback=None): + def fake_compute(paths, db_path, log_callback=None, progress_callback=None, cancel_event=None): for p in paths: fp = fake_fp(p) yield p, 0, fp From 87e744e8bff25a998c7ffcfb3a56106a59d10ecb Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sat, 20 Dec 2025 11:48:03 -0500 Subject: [PATCH 129/606] Harden library sync review with profiling and tests --- library_sync.py | 168 ++++++++++++------ main_gui.py | 44 ++--- tests/test_library_sync.py | 24 +-- tests/test_library_sync_integration.py | 227 +++++++++++++++++++++++++ tests/test_library_sync_matching.py | 100 +++++++++++ 5 files changed, 463 insertions(+), 100 deletions(-) create mode 100644 tests/test_library_sync_integration.py create mode 100644 tests/test_library_sync_matching.py diff --git a/library_sync.py b/library_sync.py index 589278f..588ab05 100644 --- a/library_sync.py +++ b/library_sync.py @@ -6,6 +6,7 @@ import logging import os import threading +import time from dataclasses import dataclass, field from enum import Enum from typing import Callable, Dict, Iterable, List, Optional, Tuple @@ -158,6 +159,45 @@ def to_dict(self) -> Dict[str, object]: } +@dataclass +class PerformanceProfile: + """Capture lightweight performance metrics for profiling runs.""" + + cache_hits: int = 0 + fingerprints_computed: int = 0 + shortlist_sizes: List[int] = field(default_factory=list) + signature_shortlists: int = 0 + extension_shortlists: int = 0 + fallback_shortlists: int = 0 + progress_updates: int = 0 + library_scan_duration: float = 0.0 + incoming_scan_duration: float = 0.0 + match_duration: float = 0.0 + + def record_shortlist(self, size: int, source: str) -> None: + self.shortlist_sizes.append(size) + if source == "signature": + self.signature_shortlists += 1 + elif source == "extension": + self.extension_shortlists += 1 + else: + self.fallback_shortlists += 1 + + def to_dict(self) -> Dict[str, object]: + return { + "cache_hits": self.cache_hits, + "fingerprints_computed": self.fingerprints_computed, + "shortlist_sizes": list(self.shortlist_sizes), + "signature_shortlists": self.signature_shortlists, + "extension_shortlists": self.extension_shortlists, + "fallback_shortlists": self.fallback_shortlists, + "progress_updates": self.progress_updates, + "library_scan_duration": self.library_scan_duration, + "incoming_scan_duration": self.incoming_scan_duration, + "match_duration": self.match_duration, + } + + def _fingerprint_signature(fp: str | None) -> str | None: if not fp: return None @@ -202,6 +242,7 @@ def _scan_folder( log_callback: Callable[[str], None] | None = None, progress_callback: Callable[[int, int, str, str], None] | None = None, cancel_event: threading.Event | None = None, + profiler: PerformanceProfile | None = None, ) -> List[TrackRecord]: """Enumerate audio files, normalize paths, and attach cached fingerprints.""" cancel_event = cancel_event or threading.Event() @@ -251,6 +292,8 @@ def _scan_folder( total_files += 1 if progress_callback: progress_callback(total_files, 0, path, "scan") + if profiler: + profiler.progress_updates += 1 if not entries: return [] @@ -272,6 +315,8 @@ def _scan_folder( _dlog(f"DEBUG: cache hit {entry['path']}", log_callback) if log_callback: log_callback(json.dumps({"event": "cache_hit", "path": entry["path"]})) + if profiler: + profiler.cache_hits += 1 else: if row: msg = ( @@ -310,6 +355,8 @@ def fp_progress(current: int, total: int, path: str, phase: str) -> None: path = result[0] duration = result[1] if len(result) > 2 else None fp = result[-1] + if profiler: + profiler.fingerprints_computed += 1 entry = entry_map.get(path) if entry is None: continue @@ -364,15 +411,22 @@ def _shortlist_candidates( ext_index: Dict[str, List[TrackRecord]], fallback_pool: List[TrackRecord], log_callback: Callable[[str], None] | None = None, + profiler: PerformanceProfile | None = None, ) -> List[TrackRecord]: sig = incoming.signature() candidates: List[TrackRecord] = [] + shortlist_source = "fallback" if sig and sig in sig_index: candidates.extend(sig_index[sig]) + shortlist_source = "signature" if not candidates: - candidates.extend(ext_index.get(incoming.ext, [])[:SHORTLIST_FALLBACK]) + ext_candidates = ext_index.get(incoming.ext, [])[:SHORTLIST_FALLBACK] + candidates.extend(ext_candidates) + shortlist_source = "extension" if ext_candidates else shortlist_source if not candidates and fallback_pool: candidates.extend(fallback_pool[:SHORTLIST_FALLBACK]) + if profiler: + profiler.record_shortlist(len(candidates), shortlist_source) _dlog( f"DEBUG: shortlist for {incoming.path} -> {len(candidates)} candidates", log_callback, @@ -386,14 +440,18 @@ def _match_tracks( thresholds: Dict[str, float], fmt_priority: Dict[str, int], log_callback: Callable[[str], None] | None = None, + profiler: PerformanceProfile | None = None, ) -> List[MatchResult]: sig_index, ext_index = _build_candidate_index(library) fallback_pool = sorted([r for r in library if r.fingerprint], key=lambda r: r.normalized_path) results: List[MatchResult] = [] + start_time = time.perf_counter() for inc in incoming: threshold_used = _select_threshold(inc.ext, thresholds) near_miss_margin = max(NEAR_MISS_MARGIN_FLOOR, threshold_used * NEAR_MISS_MARGIN_RATIO) - candidates = _shortlist_candidates(inc, sig_index, ext_index, fallback_pool, log_callback=log_callback) + candidates = _shortlist_candidates( + inc, sig_index, ext_index, fallback_pool, log_callback=log_callback, profiler=profiler + ) cand_dist: List[CandidateDistance] = [] for cand in candidates: dist = fingerprint_distance(inc.fingerprint, cand.fingerprint) @@ -444,6 +502,8 @@ def _match_tracks( existing_score=existing_score, ) ) + if profiler: + profiler.match_duration = time.perf_counter() - start_time return results @@ -458,6 +518,8 @@ def compare_libraries( log_callback: Callable[[str], None] | None = None, progress_callback: Callable[[int, int, str, str], None] | None = None, cancel_event: threading.Event | None = None, + profiler: PerformanceProfile | None = None, + include_profile: bool = False, ) -> Dict[str, object]: """Return classification of incoming files vs. an existing library. @@ -474,6 +536,8 @@ def compare_libraries( f"library_sync: comparing {library_folder} to {incoming_folder}" ) cancel_event = cancel_event or threading.Event() + if profiler is None and include_profile: + profiler = PerformanceProfile() if thresholds is None: if threshold is not None: thresholds = {"default": threshold} @@ -481,22 +545,32 @@ def compare_libraries( thresholds = DEFAULT_FP_THRESHOLDS fmt_priority = fmt_priority or FORMAT_PRIORITY + start = time.perf_counter() lib_records = _scan_folder( library_folder, db_path, log_callback=log_callback, progress_callback=progress_callback, cancel_event=cancel_event, + profiler=profiler, ) + if profiler: + profiler.library_scan_duration = time.perf_counter() - start + start = time.perf_counter() inc_records = _scan_folder( incoming_folder, db_path, log_callback=log_callback, progress_callback=progress_callback, cancel_event=cancel_event, + profiler=profiler, ) + if profiler: + profiler.incoming_scan_duration = time.perf_counter() - start - match_results = _match_tracks(inc_records, lib_records, thresholds, fmt_priority, log_callback=log_callback) + match_results = _match_tracks( + inc_records, lib_records, thresholds, fmt_priority, log_callback=log_callback, profiler=profiler + ) new_paths: List[str] = [] existing_matches: List[Tuple[str, str]] = [] @@ -525,6 +599,8 @@ def compare_libraries( "matches": [m.to_dict() for m in match_results], "partial": cancel_event.is_set(), } + if include_profile and profiler: + result["profile"] = profiler.to_dict() record_event( "library_sync: comparison complete " f"library_existing={len(lib_records)} incoming_tracks={len(inc_records)} " @@ -532,59 +608,41 @@ def compare_libraries( ) return result -def copy_new_tracks( - new_paths: List[str], - incoming_root: str, - library_root: str, - log_callback: Callable[[str], None] | None = None, -) -> List[str]: - """Copy each path in ``new_paths`` from ``incoming_root`` into ``library_root``. - Returns the list of destination paths created. - """ - import shutil - - _dlog("DEBUG: Copying new tracks", log_callback) - dest_paths = [] - for src in new_paths: - rel = os.path.relpath(src, incoming_root) - dest = os.path.join(library_root, rel) - base, ext = os.path.splitext(dest) - idx = 1 - final = dest - while os.path.exists(final): - final = f"{base} ({idx}){ext}" - idx += 1 - os.makedirs(os.path.dirname(final), exist_ok=True) - _dlog(f"DEBUG: Copy {src} -> {final}", log_callback) - shutil.copy2(src, final) - dest_paths.append(final) - return dest_paths - - -def replace_tracks( - pairs: List[Tuple[str, str]], - backup_dirname: str = "__backup__", +def profile_compare_libraries( + library_folder: str, + incoming_folder: str, + db_path: str, + threshold: float | None = None, + fmt_priority: Dict[str, int] | None = None, + thresholds: Dict[str, float] | None = None, log_callback: Callable[[str], None] | None = None, -) -> List[str]: - """Replace library files with higher quality versions. + progress_callback: Callable[[int, int, str, str], None] | None = None, + cancel_event: threading.Event | None = None, +) -> Tuple[Dict[str, object], PerformanceProfile]: + """Run a profiled comparison and return both results and metrics.""" + profile = PerformanceProfile() + result = compare_libraries( + library_folder, + incoming_folder, + db_path, + threshold=threshold, + fmt_priority=fmt_priority, + thresholds=thresholds, + log_callback=log_callback, + progress_callback=progress_callback, + cancel_event=cancel_event, + profiler=profile, + include_profile=True, + ) + return result, profile - Each pair is ``(incoming, existing)``. The original file is moved into a - ``backup_dirname`` folder alongside the existing file before the incoming - file is moved into place. - Returns the list of replaced library paths. - """ - import shutil - - _dlog("DEBUG: Replacing tracks", log_callback) - replaced = [] - for inc, lib in pairs: - backup = os.path.join(os.path.dirname(lib), backup_dirname, os.path.basename(lib)) - os.makedirs(os.path.dirname(backup), exist_ok=True) - _dlog(f"DEBUG: Backup {lib} -> {backup}", log_callback) - shutil.move(lib, backup) - _dlog(f"DEBUG: Move {inc} -> {lib}", log_callback) - shutil.move(inc, lib) - replaced.append(lib) - return replaced +def copy_new_tracks(*_args, **_kwargs): + """Deprecated: blocked per review-first redesign.""" + raise RuntimeError("File operations are disabled in the Library Sync review tool.") + + +def replace_tracks(*_args, **_kwargs): + """Deprecated: blocked per review-first redesign.""" + raise RuntimeError("File operations are disabled in the Library Sync review tool.") diff --git a/main_gui.py b/main_gui.py index b430f4e..2b10e7c 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1547,7 +1547,6 @@ def __init__(self): self.sync_debug_var = tk.BooleanVar(value=False) self.sync_library_var = tk.StringVar(value="") self.sync_incoming_var = tk.StringVar(value="") - self.sync_auto_var = tk.BooleanVar(value=True) self.sync_new = [] self.sync_existing = [] self.sync_improved = [] @@ -2174,14 +2173,16 @@ def build_ui(self): actions = ttk.Frame(self.sync_tab) actions.pack(fill="x", padx=10, pady=(0, 10)) - ttk.Checkbutton( - actions, text="Auto-Update Playlists", variable=self.sync_auto_var - ).pack(side="left") - ttk.Button(actions, text="Copy New", command=self._copy_new_tracks).pack( + ttk.Label( + actions, + text="Library Sync actions are review-only; exporting plans is non-destructive.", + wraplength=360, + ).pack(side="left", fill="x", expand=True) + ttk.Button(actions, text="Copy New", command=self._show_sync_review_notice).pack( side="left", padx=(5, 0) ) ttk.Button( - actions, text="Replace Selected", command=self._replace_selected + actions, text="Replace Selected", command=self._show_sync_review_notice ).pack(side="left", padx=(5, 0)) # after your other tabs @@ -3666,28 +3667,19 @@ def _render_sync_results(self): self.sync_improved_list.insert("end", os.path.basename(inc)) def _copy_new_tracks(self): - idxs = self.sync_new_list.curselection() - if not idxs: - return - sels = [self.sync_new[int(i)] for i in idxs] - dests = library_sync.copy_new_tracks( - sels, - self.sync_incoming_var.get(), - self.sync_library_var.get(), - ) - if self.sync_auto_var.get(): - playlist_generator.update_playlists(dests) - messagebox.showinfo("Copy New", f"Copied {len(dests)} files") + self._show_sync_review_notice() def _replace_selected(self): - idxs = self.sync_improved_list.curselection() - if not idxs: - return - sels = [self.sync_improved[int(i)] for i in idxs] - dests = library_sync.replace_tracks(sels) - if self.sync_auto_var.get(): - playlist_generator.update_playlists(dests) - messagebox.showinfo("Replace", f"Replaced {len(dests)} files") + self._show_sync_review_notice() + + def _show_sync_review_notice(self) -> None: + messagebox.showinfo( + "Review Only", + ( + "Library Sync no longer performs copy or replace operations. " + "Export the review plan instead and apply changes manually." + ), + ) # ── Quality Checker Helpers ───────────────────────────────────────── def clear_quality_view(self) -> None: diff --git a/tests/test_library_sync.py b/tests/test_library_sync.py index 153e76a..a1a5b93 100644 --- a/tests/test_library_sync.py +++ b/tests/test_library_sync.py @@ -1,6 +1,7 @@ import types import sys import os +import pytest # Stub mutagen and fingerprint generator mutagen_stub = types.ModuleType('mutagen') @@ -132,25 +133,10 @@ def fake_compute(paths, db_path, log_callback=None, progress_callback=None, canc def test_copy_and_replace(tmp_path): - lib = tmp_path / 'lib' - inc = tmp_path / 'inc' - lib.mkdir() - inc.mkdir() - f_new = inc / 'new.flac' - f_new.write_text('x') - copied = library_sync.copy_new_tracks([str(f_new)], str(inc), str(lib)) - assert len(copied) == 1 - assert os.path.exists(copied[0]) - - f_old = lib / 'old.mp3' - f_old.write_text('old') - f_better = inc / 'old.flac' - f_better.write_text('better') - replaced = library_sync.replace_tracks([(str(f_better), str(f_old))]) - assert replaced == [str(f_old)] - backup = lib / '__backup__' / 'old.mp3' - assert backup.exists() - assert f_old.read_text() == 'better' + with pytest.raises(RuntimeError): + library_sync.copy_new_tracks(["/tmp/a.flac"], "/tmp/inc", "/tmp/lib") + with pytest.raises(RuntimeError): + library_sync.replace_tracks([("/tmp/inc.flac", "/tmp/lib.mp3")]) def test_format_threshold_match(tmp_path, monkeypatch): diff --git a/tests/test_library_sync_integration.py b/tests/test_library_sync_integration.py new file mode 100644 index 0000000..3bd3dea --- /dev/null +++ b/tests/test_library_sync_integration.py @@ -0,0 +1,227 @@ +import csv +import os +import sys +import threading +import types + +import pytest + +import library_sync +from fingerprint_cache import flush_cache +from library_sync import MatchStatus, PerformanceProfile +from library_sync_review_report import ReportFormat, export_report +from library_sync_review_state import ReviewStateStore +import music_indexer_api + + +def _ensure_fp_module() -> types.ModuleType: + if "fingerprint_generator" not in sys.modules: + sys.modules["fingerprint_generator"] = types.ModuleType("fingerprint_generator") + return sys.modules["fingerprint_generator"] + + +def _setup_media_stubs(monkeypatch, fp_map, corrupt=None, missing=None, cancel_after=None): + corrupt = corrupt or set() + missing = missing or set() + fp_mod = _ensure_fp_module() + + def fake_compute(paths, db_path, log_callback=None, progress_callback=None, cancel_event=None): + for idx, path in enumerate(paths): + if cancel_after is not None and idx >= cancel_after: + if cancel_event: + cancel_event.set() + break + fp = fp_map.get(os.path.basename(path), "0 0") + yield path, 120, fp + + monkeypatch.setattr(fp_mod, "compute_fingerprints_parallel", fake_compute) + + def fake_get_tags(path): + name = os.path.basename(path) + if name in missing: + return {"artist": None, "title": None, "album": None, "year": None, "track": None, "genre": None} + return {"artist": "Artist", "title": name, "album": "Album", "year": None, "track": None, "genre": None} + + monkeypatch.setattr(music_indexer_api, "get_tags", fake_get_tags) + + class DummyInfo: + def __init__(self): + self.bitrate = 256000 + self.sample_rate = 44100 + + def fake_mutagen(path, easy=False): + if os.path.basename(path) in corrupt: + raise IOError("corrupt file") + return types.SimpleNamespace(info=DummyInfo()) + + monkeypatch.setattr(music_indexer_api, "MutagenFile", fake_mutagen) + + +def test_partial_scan_and_cancellation_preserves_results(tmp_path, monkeypatch): + fp_map = {"lib1.flac": "1 1 1", "inc1.flac": "1 1 1", "inc2.flac": "2 2 2"} + _setup_media_stubs(monkeypatch, fp_map, cancel_after=1) + + library = tmp_path / "library" + incoming = tmp_path / "incoming" + library.mkdir() + incoming.mkdir() + (library / "lib1.flac").write_text("a") + (library / "lib2.flac").write_text("b") + (incoming / "inc1.flac").write_text("c") + (incoming / "inc2.flac").write_text("d") + + db_path = tmp_path / "fp.db" + cancel_event = threading.Event() + profile = PerformanceProfile() + res = library_sync.compare_libraries( + str(library), + str(incoming), + str(db_path), + cancel_event=cancel_event, + include_profile=True, + profiler=profile, + ) + + assert res["partial"] is True + assert len(res["library_records"]) == 2 + assert len(res["incoming_records"]) in (0, 2) + if res["matches"]: + assert res["matches"][0]["status"] in {MatchStatus.NEW.value, MatchStatus.COLLISION.value} + assert profile.progress_updates > 0 + flush_cache(str(db_path)) + + +def test_rescan_threshold_change_uses_cache_and_profile(tmp_path, monkeypatch): + fp_map = {"song.mp3": "1 1 1 2", "song.wav": "1 1 1 1"} + _setup_media_stubs(monkeypatch, fp_map) + + library = tmp_path / "library" + incoming = tmp_path / "incoming" + library.mkdir() + incoming.mkdir() + (library / "song.mp3").write_text("lib") + (incoming / "song.wav").write_text("inc") + db_path = tmp_path / "fp.db" + + profile1 = PerformanceProfile() + res_strict = library_sync.compare_libraries( + str(library), + str(incoming), + str(db_path), + thresholds={".wav": 0.2, ".mp3": 0.2}, + include_profile=True, + profiler=profile1, + ) + + profile2 = PerformanceProfile() + res_relaxed = library_sync.compare_libraries( + str(library), + str(incoming), + str(db_path), + thresholds={".wav": 0.5, ".mp3": 0.5}, + include_profile=True, + profiler=profile2, + ) + + first_match = res_strict["matches"][0] + second_match = res_relaxed["matches"][0] + assert first_match["status"] in {MatchStatus.LOW_CONFIDENCE.value, MatchStatus.NEW.value} + assert second_match["status"] == MatchStatus.COLLISION.value + assert profile1.fingerprints_computed >= 2 + assert profile2.cache_hits >= 2 + assert profile2.fingerprints_computed == 0 + assert len(profile1.shortlist_sizes) == 1 + assert profile1.progress_updates >= 2 + flush_cache(str(db_path)) + + +def test_export_report_covers_flags_and_notes(tmp_path, monkeypatch): + fp_map = { + "dup.flac": "9 9 9", + "dup (1).flac": "9 9 9", + "song.mp3": "1 2 3 4", + "song.mp3.incoming": "4 3 2 1", + "missing.wav": "5 5 5 5", + "corrupt.flac": "8 8 8 8", + } + corrupt = {"corrupt.flac"} + missing = {"missing.wav"} + _setup_media_stubs(monkeypatch, fp_map, corrupt=corrupt, missing=missing) + + library = tmp_path / "library" + incoming = tmp_path / "incoming" + library.mkdir() + incoming.mkdir() + (library / "dup.flac").write_text("dup") + (library / "song.mp3").write_text("song-old") + (library / "corrupt.flac").write_text("bad") + (incoming / "dup (1).flac").write_text("dup-inc") + (incoming / "song.mp3").write_text("song-new") + (incoming / "missing.wav").write_text("missing") + (incoming / "notes.txt").write_text("skip me") + (incoming / "song.mp3.incoming").write_text("alt content") + + db_path = tmp_path / "fp.db" + lib_records = library_sync._scan_folder(str(library), str(db_path), cancel_event=threading.Event()) + inc_records = library_sync._scan_folder(str(incoming), str(db_path), cancel_event=threading.Event()) + + matches = library_sync._match_tracks( + inc_records, lib_records, {".wav": 0.3, ".mp3": 0.3, ".flac": 0.3, "default": 0.3}, {".flac": 3, ".mp3": 1} + ) + assert len(matches) == 3 + + flags = ReviewStateStore() + for res in matches: + if res.status in (MatchStatus.EXACT_MATCH, MatchStatus.COLLISION): + flags.flag_for_replace(res) + flags.set_note(res.incoming.track_id, "duplicate handled") + break + for res in matches: + if res.status == MatchStatus.NEW: + flags.flag_for_copy(res.incoming.track_id) + flags.set_note(res.incoming.track_id, "new track") + break + + json_path = tmp_path / "report.json" + csv_path = tmp_path / "report.csv" + export_report( + str(json_path), + matches, + flags, + {"global_threshold": 0.3, "per_format_overrides": {".flac": 0.3}}, + fmt=ReportFormat.JSON.value, + ) + export_report( + str(csv_path), + matches, + flags, + {"global_threshold": 0.3, "per_format_overrides": {".flac": 0.3}}, + fmt=ReportFormat.CSV.value, + ) + + with open(json_path, "r", encoding="utf-8") as f: + payload = f.read() + assert "schema_version" in payload + assert any(row.incoming.path.endswith("dup (1).flac") for row in matches) + + with open(csv_path, newline="", encoding="utf-8") as f: + rows = list(csv.DictReader(f)) + assert len(rows) == len(matches) + assert set(rows[0].keys()) >= set( + [ + "schema_version", + "incoming_path", + "existing_path", + "status", + "distance", + "threshold_used", + "incoming_quality_score", + "existing_quality_score", + "quality_delta", + "user_flag", + "notes", + "timestamp", + "scan_config_hash", + ] + ) + flush_cache(str(db_path)) diff --git a/tests/test_library_sync_matching.py b/tests/test_library_sync_matching.py new file mode 100644 index 0000000..d53ef54 --- /dev/null +++ b/tests/test_library_sync_matching.py @@ -0,0 +1,100 @@ +import pytest + +import library_sync +from library_sync import MatchStatus, PerformanceProfile, TrackRecord, compute_quality_score + + +def _record(name: str, fp: str, ext: str = ".flac", bitrate: int | None = 320000) -> TrackRecord: + return TrackRecord( + track_id=name, + path=f"/music/{name}{ext}", + normalized_path=f"/music/{name}{ext}", + ext=ext, + bitrate=bitrate or 0, + size=1024, + mtime=0.0, + fingerprint=fp, + tags={}, + duration=120, + ) + + +def test_classification_rules_and_best_match(monkeypatch): + dist_map = { + ("lib-exact", "lib-exact"): 0.0, + ("sig-exact", "lib-exact"): 0.0, + ("sig-close", "lib-close"): 0.2, + ("sig-close", "alt-close"): 0.5, + ("sig-near", "lib-near"): 0.34, + ("sig-new", "lib-new"): 0.9, + } + + def fake_distance(fp1, fp2): + return dist_map.get((fp1, fp2), 1.0) + + monkeypatch.setattr(library_sync, "fingerprint_distance", fake_distance) + + lib = [ + _record("lib-exact", "lib-exact"), + _record("lib-close", "lib-close"), + _record("lib-near", "lib-near"), + _record("alt-close", "alt-close"), + _record("lib-new", "lib-new"), + ] + incoming = [ + _record("inc-exact", "lib-exact"), + _record("inc-close", "sig-close"), + _record("inc-near", "sig-near"), + _record("inc-new", "sig-new"), + ] + + profile = PerformanceProfile() + matches = library_sync._match_tracks( + incoming, lib, {".flac": 0.3, "default": 0.3}, {".flac": 3}, profiler=profile + ) + + statuses = [m.status for m in matches] + assert statuses == [ + MatchStatus.EXACT_MATCH, + MatchStatus.COLLISION, + MatchStatus.LOW_CONFIDENCE, + MatchStatus.NEW, + ] + assert matches[1].existing.track_id == "lib-close" + assert matches[2].near_miss_margin == pytest.approx(0.075) + assert profile.signature_shortlists >= 1 + assert len(profile.shortlist_sizes) == len(incoming) + + +def test_threshold_precedence_and_margin_floor(monkeypatch): + dist_map = {("inc-wav", "lib-wav"): 0.025} + monkeypatch.setattr(library_sync, "fingerprint_distance", lambda a, b: dist_map.get((a, b), 1.0)) + + matches = library_sync._match_tracks( + [_record("inc-wav", "inc-wav", ext=".wav")], + [_record("lib-wav", "lib-wav", ext=".wav")], + {".wav": 0.01, "default": 0.3}, + {".wav": 2}, + ) + + assert matches[0].threshold_used == 0.01 + assert matches[0].near_miss_margin == pytest.approx(0.02) + assert matches[0].status == MatchStatus.LOW_CONFIDENCE + + +def test_quality_score_fallback_paths(monkeypatch): + fmt_priority = {".flac": 3, ".mp3": 1} + monkeypatch.setattr(library_sync, "fingerprint_distance", lambda *_a, **_k: 0.2) + + match = library_sync._match_tracks( + [_record("upgrade", "inc", ext=".flac", bitrate=None)], + [_record("existing", "lib", ext=".mp3", bitrate=None)], + {".flac": 0.3, "default": 0.3}, + fmt_priority, + )[0] + + assert compute_quality_score({"ext": ".flac", "bitrate": None}, fmt_priority) == 3 + assert compute_quality_score({"ext": ".mp3", "bitrate": None}, fmt_priority) == 1 + assert match.quality_label == "Potential Upgrade" + assert match.incoming_score == 3 + assert match.existing_score == 1 From ab4448dfd64418543a035b82f218ce44024d9e0f Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sat, 20 Dec 2025 12:35:12 -0500 Subject: [PATCH 130/606] Make review window scrollable --- library_sync_review.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/library_sync_review.py b/library_sync_review.py index a253040..6a72e17 100644 --- a/library_sync_review.py +++ b/library_sync_review.py @@ -238,8 +238,26 @@ def __init__(self, master: tk.Widget | None = None): def _build_ui(self) -> None: padding = {"padx": 10, "pady": 5} - root = ttk.Frame(self) - root.pack(fill="both", expand=True) + outer = ttk.Frame(self) + outer.pack(fill="both", expand=True) + + canvas = tk.Canvas(outer, borderwidth=0, highlightthickness=0) + scrollbar = ttk.Scrollbar(outer, orient="vertical", command=canvas.yview) + canvas.configure(yscrollcommand=scrollbar.set) + scrollbar.pack(side="right", fill="y") + canvas.pack(side="left", fill="both", expand=True) + + root = ttk.Frame(canvas) + root_id = canvas.create_window((0, 0), window=root, anchor="nw") + + def _on_frame_config(_event=None): + canvas.configure(scrollregion=canvas.bbox("all")) + + def _on_canvas_config(event): + canvas.itemconfigure(root_id, width=event.width) + + root.bind("", _on_frame_config) + canvas.bind("", _on_canvas_config) folder_frame = ttk.LabelFrame(root, text="Folders") folder_frame.pack(fill="x", **padding) From 7bd5827a8cf3ab0a46f8b89b0c525502bc31dd30 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sat, 20 Dec 2025 12:54:43 -0500 Subject: [PATCH 131/606] Replace Library Sync tab with review UI --- library_sync_review.py | 57 +++++++++++++--- main_gui.py | 145 +++-------------------------------------- 2 files changed, 56 insertions(+), 146 deletions(-) diff --git a/library_sync_review.py b/library_sync_review.py index 6a72e17..6bcb38d 100644 --- a/library_sync_review.py +++ b/library_sync_review.py @@ -7,7 +7,7 @@ from dataclasses import dataclass, field from datetime import datetime from enum import Enum -from typing import Any, Dict, Iterable, List, Sequence +from typing import Any, Callable, Dict, Iterable, List, Sequence import tkinter as tk from tkinter import filedialog, messagebox, ttk @@ -180,14 +180,23 @@ def _cancel(self) -> None: self.destroy() -class LibrarySyncReviewWindow(tk.Toplevel): - """Lightweight entry point for the redesign with persisted settings.""" +class LibrarySyncReviewPanel(ttk.Frame): + """Embeddable preview-first Library Sync UI.""" - def __init__(self, master: tk.Widget | None = None): + def __init__( + self, + master: tk.Widget | None = None, + *, + library_root: str | None = None, + incoming_root: str | None = None, + on_close: Callable[[], None] | None = None, + ): super().__init__(master) - self.title("Library Sync (Review)") - self.resizable(True, True) self.session = load_scan_session() + if library_root: + self.session.library_root = library_root + if incoming_root: + self.session.incoming_root = incoming_root self.library_var = tk.StringVar(value=self.session.library_root) self.incoming_var = tk.StringVar(value=self.session.incoming_root) @@ -195,6 +204,7 @@ def __init__(self, master: tk.Widget | None = None): self.preset_var = tk.StringVar(value=self.session.scan_config.preset_name or "") self.report_version_var = tk.StringVar(value=str(self.session.exported_report_version)) self.scan_state_var = tk.StringVar(value=self._describe_state(self.session.scan_state)) + self._on_close_callback = on_close # Scan + progress state self.library_cancel = threading.Event() @@ -233,7 +243,6 @@ def __init__(self, master: tk.Widget | None = None): self.LOG_LIMIT = 400 self._build_ui() - self.protocol("WM_DELETE_WINDOW", self._on_close) def _build_ui(self) -> None: padding = {"padx": 10, "pady": 5} @@ -485,8 +494,22 @@ def preview_report(self, summary: dict[str, int]) -> bool: return ReportPreviewDialog.prompt(self, summary) def _on_close(self) -> None: - if self._persist_session(): - self.destroy() + if self._persist_session() and self._on_close_callback: + self._on_close_callback() + + def set_folders(self, library_root: str | None = None, incoming_root: str | None = None) -> None: + """Update the selected folders, persisting them to the scan session.""" + changed = False + if library_root: + self.library_var.set(library_root) + self.session.library_root = library_root + changed = True + if incoming_root: + self.incoming_var.set(incoming_root) + self.session.incoming_root = incoming_root + changed = True + if changed: + save_scan_session(self.session) # ── Scan + Progress ------------------------------------------------- def _cancel_event_for(self, kind: str) -> threading.Event: @@ -799,3 +822,19 @@ def _log_scan_message(self, kind: str, msg: object) -> None: self._log_event(f"{kind}_scan", msg.get("event", "scan"), msg) else: self._log_event(f"{kind}_scan", str(msg)) + + +class LibrarySyncReviewWindow(tk.Toplevel): + """Toplevel wrapper that hosts the embeddable review panel.""" + + def __init__(self, master: tk.Widget | None = None): + super().__init__(master) + self.title("Library Sync (Review)") + self.resizable(True, True) + self.panel = LibrarySyncReviewPanel(self, on_close=self._on_close) + self.panel.pack(fill="both", expand=True) + self.protocol("WM_DELETE_WINDOW", self._on_close) + + def _on_close(self) -> None: + if self.panel._persist_session(): + self.destroy() diff --git a/main_gui.py b/main_gui.py index 2b10e7c..9905610 100644 --- a/main_gui.py +++ b/main_gui.py @@ -53,7 +53,6 @@ from tag_fixer import MIN_INTERACTIVE_SCORE, FileRecord from typing import Any, Callable, List from indexer_control import cancel_event, IndexCancelled -import library_sync import library_sync_review import playlist_generator import crash_watcher @@ -1543,19 +1542,11 @@ def __init__(self): self.plugin_views: dict[str, ttk.Frame] = {} self.active_plugin: str | None = None - # Library Sync state - self.sync_debug_var = tk.BooleanVar(value=False) - self.sync_library_var = tk.StringVar(value="") - self.sync_incoming_var = tk.StringVar(value="") - self.sync_new = [] - self.sync_existing = [] - self.sync_improved = [] - self.sync_existing_matches = [] - self.sync_unmatched_new = [] self.use_review_sync_var = tk.BooleanVar( value=cfg.get("use_library_sync_review", False) ) self.sync_review_window: library_sync_review.LibrarySyncReviewWindow | None = None + self.sync_review_panel: library_sync_review.LibrarySyncReviewPanel | None = None # ── Tag Fixer state ── self.tagfix_folder_var = tk.StringVar(value="") @@ -2129,61 +2120,11 @@ def build_ui(self): # ─── Library Sync Tab ───────────────────────────────────────────── self.sync_tab = ttk.Frame(self.notebook) self.notebook.add(self.sync_tab, text="Library Sync") - - path_row = ttk.Frame(self.sync_tab) - path_row.pack(fill="x", padx=10, pady=(10, 5)) - ttk.Label(path_row, text="Library Folder:").grid(row=0, column=0, sticky="w") - ttk.Entry(path_row, textvariable=self.sync_library_var, state="readonly").grid( - row=0, column=1, sticky="ew" - ) - ttk.Button(path_row, text="Browse…", command=self._browse_sync_library).grid( - row=0, column=2, padx=(5, 0) - ) - path_row.columnconfigure(1, weight=1) - - inc_row = ttk.Frame(self.sync_tab) - inc_row.pack(fill="x", padx=10, pady=(0, 5)) - ttk.Label(inc_row, text="Incoming Folder:").grid(row=0, column=0, sticky="w") - ttk.Entry(inc_row, textvariable=self.sync_incoming_var).grid( - row=0, column=1, sticky="ew" - ) - ttk.Button(inc_row, text="Browse…", command=self._browse_sync_incoming).grid( - row=0, column=2, padx=(5, 0) - ) - inc_row.columnconfigure(1, weight=1) - - ttk.Button(self.sync_tab, text="Scan", command=self._scan_library_sync).pack( - pady=5 - ) - - lists = ttk.Frame(self.sync_tab) - lists.pack(fill="both", expand=True, padx=10, pady=5) - lists.columnconfigure((0, 1, 2), weight=1) - - ttk.Label(lists, text="New Tracks").grid(row=0, column=0) - ttk.Label(lists, text="Existing").grid(row=0, column=1) - ttk.Label(lists, text="Improvement Candidates").grid(row=0, column=2) - - self.sync_new_list = tk.Listbox(lists, selectmode="extended") - self.sync_new_list.grid(row=1, column=0, sticky="nsew") - self.sync_existing_list = tk.Listbox(lists, selectmode="extended") - self.sync_existing_list.grid(row=1, column=1, sticky="nsew") - self.sync_improved_list = tk.Listbox(lists, selectmode="extended") - self.sync_improved_list.grid(row=1, column=2, sticky="nsew") - - actions = ttk.Frame(self.sync_tab) - actions.pack(fill="x", padx=10, pady=(0, 10)) - ttk.Label( - actions, - text="Library Sync actions are review-only; exporting plans is non-destructive.", - wraplength=360, - ).pack(side="left", fill="x", expand=True) - ttk.Button(actions, text="Copy New", command=self._show_sync_review_notice).pack( - side="left", padx=(5, 0) + self.sync_review_panel = library_sync_review.LibrarySyncReviewPanel( + self.sync_tab, + library_root=self.library_path or "", ) - ttk.Button( - actions, text="Replace Selected", command=self._show_sync_review_notice - ).pack(side="left", padx=(5, 0)) + self.sync_review_panel.pack(fill="both", expand=True) # after your other tabs help_frame = ttk.Frame(self.notebook) @@ -2296,6 +2237,8 @@ def select_library(self): if hasattr(self, "player_reload_btn"): self.player_reload_btn.config(state="normal") self._load_player_library_async() + if self.sync_review_panel: + self.sync_review_panel.set_folders(library_root=self.library_path) def update_library_info(self): if not self.library_path: @@ -3606,81 +3549,9 @@ def _open_library_sync_tool(self) -> None: self.notebook.select(self.sync_tab) except Exception: messagebox.showinfo( - "Library Sync", "The classic Library Sync tab is not available." + "Library Sync", "The Library Sync tab is not available." ) - def _browse_sync_library(self): - initial = self.sync_library_var.get() or load_last_path() - folder = filedialog.askdirectory( - title="Select Library Folder", initialdir=initial - ) - if folder: - self.sync_library_var.set(folder) - - def _browse_sync_incoming(self): - initial = self.sync_incoming_var.get() or load_last_path() - folder = filedialog.askdirectory( - title="Select Incoming Folder", initialdir=initial - ) - if folder: - self.sync_incoming_var.set(folder) - - def _scan_library_sync(self): - lib = self.sync_library_var.get() - inc = self.sync_incoming_var.get() - if not lib or not inc: - messagebox.showwarning( - "Scan", "Please choose library and incoming folders." - ) - return - db = os.path.join(lib, "Docs", ".soundvault.db") - cfg = load_config() - thresholds = cfg.get("format_fp_thresholds", DEFAULT_FP_THRESHOLDS) - - library_sync.set_debug(self.sync_debug_var.get()) - - def task(): - try: - res = library_sync.compare_libraries( - lib, inc, db, thresholds=thresholds - ) - self.sync_new = res["new_tracks"] - self.sync_existing = res["existing"] - self.sync_improved = res["improved"] - self.sync_existing_matches = res.get("existing_matches", []) - self.sync_unmatched_new = res.get("new", []) - self.after(0, self._render_sync_results) - except Exception as e: - self.after(0, lambda: messagebox.showerror("Scan Failed", str(e))) - - threading.Thread(target=task, daemon=True).start() - - def _render_sync_results(self): - self.sync_new_list.delete(0, "end") - self.sync_existing_list.delete(0, "end") - self.sync_improved_list.delete(0, "end") - for p in self.sync_new: - self.sync_new_list.insert("end", os.path.basename(p)) - for lib in self.sync_existing: - self.sync_existing_list.insert("end", os.path.basename(lib)) - for inc, _lib in self.sync_improved: - self.sync_improved_list.insert("end", os.path.basename(inc)) - - def _copy_new_tracks(self): - self._show_sync_review_notice() - - def _replace_selected(self): - self._show_sync_review_notice() - - def _show_sync_review_notice(self) -> None: - messagebox.showinfo( - "Review Only", - ( - "Library Sync no longer performs copy or replace operations. " - "Export the review plan instead and apply changes manually." - ), - ) - # ── Quality Checker Helpers ───────────────────────────────────────── def clear_quality_view(self) -> None: for w in self.qc_inner.winfo_children(): From 65664185127b91b92a1ff7c5ad87f3415220e338 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sat, 20 Dec 2025 13:05:04 -0500 Subject: [PATCH 132/606] Remove legacy Library Sync tab and toggles --- config.py | 2 -- library_sync_review.py | 14 --------- main_gui.py | 65 +++++++++++++----------------------------- 3 files changed, 20 insertions(+), 61 deletions(-) diff --git a/config.py b/config.py index ac9abad..4750beb 100644 --- a/config.py +++ b/config.py @@ -60,7 +60,6 @@ def load_config(): cfg.setdefault("fingerprint_duration_ms", FP_DURATION_MS) cfg.setdefault("allow_mismatched_edits", ALLOW_MISMATCHED_EDITS) cfg.setdefault("library_root", "") - cfg.setdefault("use_library_sync_review", False) cfg.setdefault("library_sync_review", {}) return cfg except Exception: @@ -71,7 +70,6 @@ def load_config(): "library_root": "", "duplicate_threshold": DEFAULT_DUP_THRESHOLD, "duplicate_prefix_len": DEFAULT_DUP_PREFIX_LEN, - "use_library_sync_review": False, "library_sync_review": {}, } diff --git a/library_sync_review.py b/library_sync_review.py index 6bcb38d..bd117d8 100644 --- a/library_sync_review.py +++ b/library_sync_review.py @@ -19,7 +19,6 @@ from library_sync_review_state import ReviewStateStore REVIEW_CONFIG_KEY = "library_sync_review" -REVIEW_FLAG_KEY = "use_library_sync_review" class ScanState(str, Enum): @@ -111,19 +110,6 @@ def save_scan_session(session: ScanSession) -> None: save_config(cfg) -def is_review_enabled() -> bool: - """Return whether the review-first entry point is enabled.""" - cfg = load_config() - return bool(cfg.get(REVIEW_FLAG_KEY, False)) - - -def set_review_enabled(enabled: bool) -> None: - """Toggle the review-first entry point flag.""" - cfg = load_config() - cfg[REVIEW_FLAG_KEY] = bool(enabled) - save_config(cfg) - - class ReportPreviewDialog(tk.Toplevel): """Simple dialog that summarizes report counts before export.""" diff --git a/main_gui.py b/main_gui.py index 9905610..0c93041 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1542,11 +1542,7 @@ def __init__(self): self.plugin_views: dict[str, ttk.Frame] = {} self.active_plugin: str | None = None - self.use_review_sync_var = tk.BooleanVar( - value=cfg.get("use_library_sync_review", False) - ) self.sync_review_window: library_sync_review.LibrarySyncReviewWindow | None = None - self.sync_review_panel: library_sync_review.LibrarySyncReviewPanel | None = None # ── Tag Fixer state ── self.tagfix_folder_var = tk.StringVar(value="") @@ -1656,12 +1652,7 @@ def build_ui(self): tools_menu = tk.Menu(menubar, tearoff=False) tools_menu.add_command( - label="Library Sync…", command=self._open_library_sync_tool - ) - tools_menu.add_checkbutton( - label="Use Library Sync (Review)", - variable=self.use_review_sync_var, - command=self._toggle_library_sync_mode, + label="Library Sync (Preview)…", command=self._open_library_sync_tool ) tools_menu.add_separator() tools_menu.add_command(label="Fix Tags via AcoustID", command=self.fix_tags_gui) @@ -2117,15 +2108,6 @@ def build_ui(self): ).pack(side="left", expand=True, fill="x", padx=(5, 0)) self._sync_player_playlist() - # ─── Library Sync Tab ───────────────────────────────────────────── - self.sync_tab = ttk.Frame(self.notebook) - self.notebook.add(self.sync_tab, text="Library Sync") - self.sync_review_panel = library_sync_review.LibrarySyncReviewPanel( - self.sync_tab, - library_root=self.library_path or "", - ) - self.sync_review_panel.pack(fill="both", expand=True) - # after your other tabs help_frame = ttk.Frame(self.notebook) self.notebook.add(help_frame, text="Help") @@ -2237,8 +2219,8 @@ def select_library(self): if hasattr(self, "player_reload_btn"): self.player_reload_btn.config(state="normal") self._load_player_library_async() - if self.sync_review_panel: - self.sync_review_panel.set_folders(library_root=self.library_path) + if self.sync_review_window and self.sync_review_window.winfo_exists(): + self.sync_review_window.panel.set_folders(library_root=self.library_path) def update_library_info(self): if not self.library_path: @@ -3524,33 +3506,26 @@ def reset_tagfix_log(self): self._log(f"Reset tag-fix log for {folder}") # ── Library Sync Helpers ─────────────────────────────────────────── - def _toggle_library_sync_mode(self) -> None: - cfg = load_config() - cfg["use_library_sync_review"] = bool(self.use_review_sync_var.get()) - save_config(cfg) - def _open_library_sync_tool(self) -> None: - if self.use_review_sync_var.get(): - if self.sync_review_window and self.sync_review_window.winfo_exists(): - self.sync_review_window.lift() - self.sync_review_window.focus_set() - return - try: - self.sync_review_window = library_sync_review.LibrarySyncReviewWindow( - self - ) - self.sync_review_window.bind( - "", lambda _e: setattr(self, "sync_review_window", None) + if self.sync_review_window and self.sync_review_window.winfo_exists(): + if self.library_path: + self.sync_review_window.panel.set_folders( + library_root=self.library_path ) - except Exception as exc: - messagebox.showerror("Library Sync (Review)", str(exc)) - else: - try: - self.notebook.select(self.sync_tab) - except Exception: - messagebox.showinfo( - "Library Sync", "The Library Sync tab is not available." + self.sync_review_window.lift() + self.sync_review_window.focus_set() + return + try: + self.sync_review_window = library_sync_review.LibrarySyncReviewWindow(self) + if self.library_path: + self.sync_review_window.panel.set_folders( + library_root=self.library_path ) + self.sync_review_window.bind( + "", lambda _e: setattr(self, "sync_review_window", None) + ) + except Exception as exc: + messagebox.showerror("Library Sync (Review)", str(exc)) # ── Quality Checker Helpers ───────────────────────────────────────── def clear_quality_view(self) -> None: From b12e32fcd6eef5df65fc7f3dc1c212d5328be8e0 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sat, 20 Dec 2025 13:06:13 -0500 Subject: [PATCH 133/606] Revert "Remove legacy Library Sync tab and toggles" --- config.py | 2 ++ library_sync_review.py | 14 +++++++++ main_gui.py | 65 +++++++++++++++++++++++++++++------------- 3 files changed, 61 insertions(+), 20 deletions(-) diff --git a/config.py b/config.py index 4750beb..ac9abad 100644 --- a/config.py +++ b/config.py @@ -60,6 +60,7 @@ def load_config(): cfg.setdefault("fingerprint_duration_ms", FP_DURATION_MS) cfg.setdefault("allow_mismatched_edits", ALLOW_MISMATCHED_EDITS) cfg.setdefault("library_root", "") + cfg.setdefault("use_library_sync_review", False) cfg.setdefault("library_sync_review", {}) return cfg except Exception: @@ -70,6 +71,7 @@ def load_config(): "library_root": "", "duplicate_threshold": DEFAULT_DUP_THRESHOLD, "duplicate_prefix_len": DEFAULT_DUP_PREFIX_LEN, + "use_library_sync_review": False, "library_sync_review": {}, } diff --git a/library_sync_review.py b/library_sync_review.py index bd117d8..6bcb38d 100644 --- a/library_sync_review.py +++ b/library_sync_review.py @@ -19,6 +19,7 @@ from library_sync_review_state import ReviewStateStore REVIEW_CONFIG_KEY = "library_sync_review" +REVIEW_FLAG_KEY = "use_library_sync_review" class ScanState(str, Enum): @@ -110,6 +111,19 @@ def save_scan_session(session: ScanSession) -> None: save_config(cfg) +def is_review_enabled() -> bool: + """Return whether the review-first entry point is enabled.""" + cfg = load_config() + return bool(cfg.get(REVIEW_FLAG_KEY, False)) + + +def set_review_enabled(enabled: bool) -> None: + """Toggle the review-first entry point flag.""" + cfg = load_config() + cfg[REVIEW_FLAG_KEY] = bool(enabled) + save_config(cfg) + + class ReportPreviewDialog(tk.Toplevel): """Simple dialog that summarizes report counts before export.""" diff --git a/main_gui.py b/main_gui.py index 0c93041..9905610 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1542,7 +1542,11 @@ def __init__(self): self.plugin_views: dict[str, ttk.Frame] = {} self.active_plugin: str | None = None + self.use_review_sync_var = tk.BooleanVar( + value=cfg.get("use_library_sync_review", False) + ) self.sync_review_window: library_sync_review.LibrarySyncReviewWindow | None = None + self.sync_review_panel: library_sync_review.LibrarySyncReviewPanel | None = None # ── Tag Fixer state ── self.tagfix_folder_var = tk.StringVar(value="") @@ -1652,7 +1656,12 @@ def build_ui(self): tools_menu = tk.Menu(menubar, tearoff=False) tools_menu.add_command( - label="Library Sync (Preview)…", command=self._open_library_sync_tool + label="Library Sync…", command=self._open_library_sync_tool + ) + tools_menu.add_checkbutton( + label="Use Library Sync (Review)", + variable=self.use_review_sync_var, + command=self._toggle_library_sync_mode, ) tools_menu.add_separator() tools_menu.add_command(label="Fix Tags via AcoustID", command=self.fix_tags_gui) @@ -2108,6 +2117,15 @@ def build_ui(self): ).pack(side="left", expand=True, fill="x", padx=(5, 0)) self._sync_player_playlist() + # ─── Library Sync Tab ───────────────────────────────────────────── + self.sync_tab = ttk.Frame(self.notebook) + self.notebook.add(self.sync_tab, text="Library Sync") + self.sync_review_panel = library_sync_review.LibrarySyncReviewPanel( + self.sync_tab, + library_root=self.library_path or "", + ) + self.sync_review_panel.pack(fill="both", expand=True) + # after your other tabs help_frame = ttk.Frame(self.notebook) self.notebook.add(help_frame, text="Help") @@ -2219,8 +2237,8 @@ def select_library(self): if hasattr(self, "player_reload_btn"): self.player_reload_btn.config(state="normal") self._load_player_library_async() - if self.sync_review_window and self.sync_review_window.winfo_exists(): - self.sync_review_window.panel.set_folders(library_root=self.library_path) + if self.sync_review_panel: + self.sync_review_panel.set_folders(library_root=self.library_path) def update_library_info(self): if not self.library_path: @@ -3506,26 +3524,33 @@ def reset_tagfix_log(self): self._log(f"Reset tag-fix log for {folder}") # ── Library Sync Helpers ─────────────────────────────────────────── + def _toggle_library_sync_mode(self) -> None: + cfg = load_config() + cfg["use_library_sync_review"] = bool(self.use_review_sync_var.get()) + save_config(cfg) + def _open_library_sync_tool(self) -> None: - if self.sync_review_window and self.sync_review_window.winfo_exists(): - if self.library_path: - self.sync_review_window.panel.set_folders( - library_root=self.library_path + if self.use_review_sync_var.get(): + if self.sync_review_window and self.sync_review_window.winfo_exists(): + self.sync_review_window.lift() + self.sync_review_window.focus_set() + return + try: + self.sync_review_window = library_sync_review.LibrarySyncReviewWindow( + self ) - self.sync_review_window.lift() - self.sync_review_window.focus_set() - return - try: - self.sync_review_window = library_sync_review.LibrarySyncReviewWindow(self) - if self.library_path: - self.sync_review_window.panel.set_folders( - library_root=self.library_path + self.sync_review_window.bind( + "", lambda _e: setattr(self, "sync_review_window", None) + ) + except Exception as exc: + messagebox.showerror("Library Sync (Review)", str(exc)) + else: + try: + self.notebook.select(self.sync_tab) + except Exception: + messagebox.showinfo( + "Library Sync", "The Library Sync tab is not available." ) - self.sync_review_window.bind( - "", lambda _e: setattr(self, "sync_review_window", None) - ) - except Exception as exc: - messagebox.showerror("Library Sync (Review)", str(exc)) # ── Quality Checker Helpers ───────────────────────────────────────── def clear_quality_view(self) -> None: From 5dd22157e2302b087aa7f9df2ec73100d554ee97 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sat, 20 Dec 2025 13:18:28 -0500 Subject: [PATCH 134/606] Add indexer engine baseline for Library Sync --- library_sync.py | 19 +- library_sync_indexer_engine/__init__.py | 37 + .../indexer_engine/__init__.py | 1 + .../indexer_engine/config.py | 82 ++ .../indexer_engine/crash_logger.py | 159 +++ .../indexer_engine/crash_watcher.py | 92 ++ .../indexer_engine/dry_run_coordinator.py | 42 + .../indexer_engine/fingerprint_cache.py | 122 ++ .../indexer_engine/fingerprint_generator.py | 224 ++++ .../indexer_engine/indexer_control.py | 13 + .../indexer_engine/music_indexer_api.py | 1174 +++++++++++++++++ .../indexer_engine/near_duplicate_detector.py | 256 ++++ .../indexer_engine/playlist_generator.py | 188 +++ .../indexer_engine/utils/path_helpers.py | 20 + 14 files changed, 2424 insertions(+), 5 deletions(-) create mode 100644 library_sync_indexer_engine/__init__.py create mode 100644 library_sync_indexer_engine/indexer_engine/__init__.py create mode 100644 library_sync_indexer_engine/indexer_engine/config.py create mode 100644 library_sync_indexer_engine/indexer_engine/crash_logger.py create mode 100644 library_sync_indexer_engine/indexer_engine/crash_watcher.py create mode 100644 library_sync_indexer_engine/indexer_engine/dry_run_coordinator.py create mode 100644 library_sync_indexer_engine/indexer_engine/fingerprint_cache.py create mode 100644 library_sync_indexer_engine/indexer_engine/fingerprint_generator.py create mode 100644 library_sync_indexer_engine/indexer_engine/indexer_control.py create mode 100644 library_sync_indexer_engine/indexer_engine/music_indexer_api.py create mode 100644 library_sync_indexer_engine/indexer_engine/near_duplicate_detector.py create mode 100644 library_sync_indexer_engine/indexer_engine/playlist_generator.py create mode 100644 library_sync_indexer_engine/indexer_engine/utils/path_helpers.py diff --git a/library_sync.py b/library_sync.py index 588ab05..2686fec 100644 --- a/library_sync.py +++ b/library_sync.py @@ -11,11 +11,20 @@ from enum import Enum from typing import Callable, Dict, Iterable, List, Optional, Tuple -import music_indexer_api as idx -import fingerprint_generator -from fingerprint_cache import _ensure_db -from near_duplicate_detector import fingerprint_distance -from config import FORMAT_PRIORITY, DEFAULT_FP_THRESHOLDS +from library_sync_indexer_engine import ( + config as idx_config, + fingerprint_cache as idx_fingerprint_cache, + fingerprint_generator as idx_fingerprint_generator, + music_indexer_api as idx, + near_duplicate_detector as idx_near_duplicate_detector, +) + +fingerprint_generator = idx_fingerprint_generator +fingerprint_distance = idx_near_duplicate_detector.fingerprint_distance +_ensure_db = idx_fingerprint_cache._ensure_db +FORMAT_PRIORITY = idx_config.FORMAT_PRIORITY +DEFAULT_FP_THRESHOLDS = idx_config.DEFAULT_FP_THRESHOLDS + from crash_watcher import record_event from crash_logger import watcher diff --git a/library_sync_indexer_engine/__init__.py b/library_sync_indexer_engine/__init__.py new file mode 100644 index 0000000..b2f0081 --- /dev/null +++ b/library_sync_indexer_engine/__init__.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import importlib +import sys +from pathlib import Path + +_ENGINE_DIR = Path(__file__).resolve().parent / "indexer_engine" +_ENGINE_DIR_STR = str(_ENGINE_DIR) +if _ENGINE_DIR_STR not in sys.path: + sys.path.insert(0, _ENGINE_DIR_STR) + + +def _load(name: str): + return importlib.import_module(name) + + +music_indexer_api = _load("music_indexer_api") +dry_run_coordinator = _load("dry_run_coordinator") +config = _load("config") +indexer_control = _load("indexer_control") +fingerprint_cache = _load("fingerprint_cache") +fingerprint_generator = _load("fingerprint_generator") +near_duplicate_detector = _load("near_duplicate_detector") +playlist_generator = _load("playlist_generator") +crash_logger = _load("crash_logger") + +__all__ = [ + "music_indexer_api", + "dry_run_coordinator", + "config", + "indexer_control", + "fingerprint_cache", + "fingerprint_generator", + "near_duplicate_detector", + "playlist_generator", + "crash_logger", +] diff --git a/library_sync_indexer_engine/indexer_engine/__init__.py b/library_sync_indexer_engine/indexer_engine/__init__.py new file mode 100644 index 0000000..d67dd77 --- /dev/null +++ b/library_sync_indexer_engine/indexer_engine/__init__.py @@ -0,0 +1 @@ +# Indexer engine baseline for Library Sync. diff --git a/library_sync_indexer_engine/indexer_engine/config.py b/library_sync_indexer_engine/indexer_engine/config.py new file mode 100644 index 0000000..ac9abad --- /dev/null +++ b/library_sync_indexer_engine/indexer_engine/config.py @@ -0,0 +1,82 @@ +import os +import json + +CONFIG_PATH = os.path.expanduser("~/.soundvault_config.json") + +# List of external metadata services supported by the application. +SUPPORTED_SERVICES = [ + "AcoustID", + "Last.fm", + "Spotify", + "MusicBrainz", + "Gracenote", +] + +# Threshold for considering two tracks as near-duplicates when syncing +NEAR_DUPLICATE_THRESHOLD = 0.1 + +# File format quality priority used during Library Sync +FORMAT_PRIORITY = {".flac": 3, ".wav": 2, ".mp3": 1} + +# Default fingerprint matching thresholds by file extension. The ``default`` key +# is used when a specific extension is not provided. +DEFAULT_FP_THRESHOLDS = { + "default": 0.3, + ".flac": 0.3, + ".mp3": 0.3, + ".aac": 0.3, +} + +# Default threshold for simple duplicate detection +DEFAULT_DUP_THRESHOLD = 0.03 +# Default prefix length used for duplicate grouping +DEFAULT_DUP_PREFIX_LEN = 16 + +# Default settings for fingerprint normalization +FP_OFFSET_MS = 0 +FP_DURATION_MS = 120_000 +ALLOW_MISMATCHED_EDITS = True + + +def load_config(): + """Load configuration from ``CONFIG_PATH``. + + Returns an empty dict if the file does not exist or can't be read. + """ + try: + with open(CONFIG_PATH, "r", encoding="utf-8") as f: + cfg = json.load(f) + if "musicbrainz_useragent" not in cfg: + cfg["musicbrainz_useragent"] = { + "app": "", + "version": "", + "contact": "", + } + if "format_fp_thresholds" not in cfg: + cfg["format_fp_thresholds"] = DEFAULT_FP_THRESHOLDS.copy() + cfg.setdefault("duplicate_threshold", DEFAULT_DUP_THRESHOLD) + cfg.setdefault("duplicate_prefix_len", DEFAULT_DUP_PREFIX_LEN) + cfg.setdefault("fingerprint_offset_ms", FP_OFFSET_MS) + cfg.setdefault("fingerprint_duration_ms", FP_DURATION_MS) + cfg.setdefault("allow_mismatched_edits", ALLOW_MISMATCHED_EDITS) + cfg.setdefault("library_root", "") + cfg.setdefault("use_library_sync_review", False) + cfg.setdefault("library_sync_review", {}) + return cfg + except Exception: + return { + "fingerprint_offset_ms": FP_OFFSET_MS, + "fingerprint_duration_ms": FP_DURATION_MS, + "allow_mismatched_edits": ALLOW_MISMATCHED_EDITS, + "library_root": "", + "duplicate_threshold": DEFAULT_DUP_THRESHOLD, + "duplicate_prefix_len": DEFAULT_DUP_PREFIX_LEN, + "use_library_sync_review": False, + "library_sync_review": {}, + } + + +def save_config(cfg: dict) -> None: + """Write ``cfg`` to ``CONFIG_PATH`` as JSON.""" + with open(CONFIG_PATH, "w", encoding="utf-8") as f: + json.dump(cfg, f, indent=2) diff --git a/library_sync_indexer_engine/indexer_engine/crash_logger.py b/library_sync_indexer_engine/indexer_engine/crash_logger.py new file mode 100644 index 0000000..b486923 --- /dev/null +++ b/library_sync_indexer_engine/indexer_engine/crash_logger.py @@ -0,0 +1,159 @@ +"""Enhanced crash logger with context and threaded exception support.""" + +from __future__ import annotations + +import logging +from logging.handlers import RotatingFileHandler +import os +import platform +import functools +import time +try: + import resource +except ModuleNotFoundError: # resource module unavailable on Windows + resource = None +import shutil +import sys +import threading +import traceback + +from typing import Callable, Dict, List, TypeVar +from typing_extensions import ParamSpec + +import crash_watcher + +try: + import tkinter as tk + from tkinter import messagebox +except Exception: # pragma: no cover - tkinter may be unavailable + tk = None # type: ignore + messagebox = None # type: ignore + + +_context_providers: List[Callable[[], Dict[str, object]]] = [] + + +P = ParamSpec("P") +R = TypeVar("R") + + +class _WatcherHelper: + """Helper exposing decorators for crash event instrumentation.""" + + def traced(self, func: Callable[P, R]) -> Callable[P, R]: + """Decorator logging entry and exit of ``func``.""" + + @functools.wraps(func) + def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: + name = func.__qualname__ + crash_watcher.record_event(f"enter {name}") + try: + return func(*args, **kwargs) + finally: + crash_watcher.record_event(f"exit {name}") + + return wrapper + + +watcher = _WatcherHelper() + + +def add_context_provider(func: Callable[[], Dict[str, object]]) -> None: + """Register a callable returning additional context for crash logs.""" + + _context_providers.append(func) + + +def install(log_path: str = "crash.log", *, level: int = logging.INFO) -> None: + """Install global crash logging. + + Parameters + ---------- + log_path: + File path for the rotating crash log. + level: + Logging level for the root logger. Can be changed later via + :func:`toggle_debug_mode`. + """ + + logger = logging.getLogger() + logger.setLevel(level) + handler = RotatingFileHandler( + log_path, maxBytes=1_000_000, backupCount=5, encoding="utf-8" + ) + fmt = "%(asctime)s [%(name)s] %(levelname)s %(funcName)s:%(lineno)d - %(message)s" + handler.setFormatter(logging.Formatter(fmt)) + logger.addHandler(handler) + + _log_startup_context(logger) + + def _handle(exc_type, exc_value, exc_tb): + if issubclass(exc_type, KeyboardInterrupt): + sys.__excepthook__(exc_type, exc_value, exc_tb) + return + + stack = "".join(traceback.format_exception(exc_type, exc_value, exc_tb)) + thread_name = threading.current_thread().name + ts = time.strftime("%Y-%m-%d %H:%M:%S") + events = crash_watcher.dump_events() + context = _gather_context() + summary = ( + f"{ts} [{thread_name}] Unhandled exception:\n{stack}\nRecent events:\n{events}\nContext: {context}" + ) + logger.critical(summary) + + if tk and messagebox: + try: + root = tk.Tk() + root.withdraw() + messagebox.showerror("Crash Report", summary) + root.destroy() + except Exception: + pass + + crash_watcher.mark_clean_shutdown() + sys.__excepthook__(exc_type, exc_value, exc_tb) + + def _handle_thread_exception(args: threading.ExceptHookArgs) -> None: + _handle(args.exc_type, args.exc_value, args.exc_traceback) + + sys.excepthook = _handle + threading.excepthook = _handle_thread_exception + + +def toggle_debug_mode() -> None: + """Raise logging level to DEBUG at runtime.""" + + logging.getLogger().setLevel(logging.DEBUG) + + +def _gather_context() -> Dict[str, object]: + """Collect runtime context information.""" + + ctx: Dict[str, object] = { + "argv": sys.argv, + "cwd": os.getcwd(), + "python": sys.version.replace("\n", " "), + "platform": platform.platform(), + "memory_kb": ( + getattr(resource.getrusage(resource.RUSAGE_SELF), "ru_maxrss", "N/A") + if resource is not None + else "N/A" + ), + "fpcalc": shutil.which("fpcalc"), + "ffmpeg": shutil.which("ffmpeg"), + } + for fn in _context_providers: + try: + ctx.update(fn()) + except Exception: + pass + return ctx + + +def _log_startup_context(logger: logging.Logger) -> None: + ctx = _gather_context() + logger.info("Application start") + for k, v in ctx.items(): + logger.info("%s: %s", k, v) + diff --git a/library_sync_indexer_engine/indexer_engine/crash_watcher.py b/library_sync_indexer_engine/indexer_engine/crash_watcher.py new file mode 100644 index 0000000..fc55c65 --- /dev/null +++ b/library_sync_indexer_engine/indexer_engine/crash_watcher.py @@ -0,0 +1,92 @@ +"""CrashWatcher module recording recent events and emitting crash reports.""" + +from __future__ import annotations + +import atexit +import threading +import time +from collections import deque +from typing import Deque + +try: + import tkinter as tk + from tkinter import messagebox +except Exception: # pragma: no cover - tkinter may be unavailable in some envs + tk = None # type: ignore + messagebox = None # type: ignore + + +class CrashWatcher(threading.Thread): + """Background thread storing recent events in a circular buffer.""" + + def __init__(self, max_events: int = 100): + super().__init__(daemon=True) + self._events: Deque[str] = deque(maxlen=max_events) + self._lock = threading.Lock() + + def record_event(self, msg: str) -> None: + ts = time.strftime("%Y-%m-%d %H:%M:%S") + thread = threading.current_thread().name + with self._lock: + self._events.append(f"{ts} [{thread}] {msg}") + + def dump_events(self) -> str: + with self._lock: + return "\n".join(self._events) + + def run(self) -> None: # pragma: no cover - thread performs no active work + while True: + time.sleep(1) + + +_watcher: CrashWatcher | None = None +clean_shutdown = False + + +def start(max_events: int = 100) -> None: + """Start the global crash watcher thread and register exit handler.""" + global _watcher + if _watcher is None: + _watcher = CrashWatcher(max_events=max_events) + _watcher.start() + atexit.register(_handle_exit) + + +def record_event(msg: str) -> None: + """Record an event in the crash buffer.""" + if _watcher is not None: + _watcher.record_event(msg) + + +def dump_events() -> str: + """Return concatenated recent events.""" + if _watcher is None: + return "" + return _watcher.dump_events() + + +def mark_clean_shutdown() -> None: + """Set the global clean shutdown flag.""" + global clean_shutdown + clean_shutdown = True + + +def _handle_exit() -> None: + if clean_shutdown or _watcher is None: + return + + report_path = "crash_report.txt" + data = _watcher.dump_events() + with open(report_path, "w", encoding="utf-8") as fh: + fh.write(data) + + if tk and messagebox: + try: + root = tk.Tk() + root.withdraw() + messagebox.showerror("Crash Report", data) + root.destroy() + except Exception: + pass + else: # pragma: no cover - in headless environments + print(f"Crash report written to {report_path}") diff --git a/library_sync_indexer_engine/indexer_engine/dry_run_coordinator.py b/library_sync_indexer_engine/indexer_engine/dry_run_coordinator.py new file mode 100644 index 0000000..f627ab3 --- /dev/null +++ b/library_sync_indexer_engine/indexer_engine/dry_run_coordinator.py @@ -0,0 +1,42 @@ +# dry_run_coordinator.py +"""Thread-safe coordinator for dry-run phases.""" +from __future__ import annotations + +import threading +from typing import List, Any, Dict + + +class DryRunCoordinator: + """Aggregate results and HTML from dry-run phases.""" + + def __init__(self) -> None: + self._lock = threading.Lock() + self.exact_dupes: List[Any] = [] + self.metadata_groups: List[Any] = [] + self.near_dupe_clusters: List[Any] = [] + self._sections: Dict[str, str] = {} + + def add_exact_dupes(self, items: List[Any]) -> None: + with self._lock: + self.exact_dupes.extend(items) + + def add_metadata_groups(self, items: List[Any]) -> None: + with self._lock: + self.metadata_groups.extend(items) + + def add_near_dupe_clusters(self, clusters: List[Any]) -> None: + with self._lock: + self.near_dupe_clusters.extend(clusters) + + def set_html_section(self, phase: str, html: str) -> None: + with self._lock: + self._sections[phase] = html or "" + + def assemble_final_report(self) -> str: + with self._lock: + parts = [] + for p in ["A", "B", "C"]: + section = self._sections.get(p) + if section: + parts.append(section) + return "\n".join(parts) diff --git a/library_sync_indexer_engine/indexer_engine/fingerprint_cache.py b/library_sync_indexer_engine/indexer_engine/fingerprint_cache.py new file mode 100644 index 0000000..1f1460b --- /dev/null +++ b/library_sync_indexer_engine/indexer_engine/fingerprint_cache.py @@ -0,0 +1,122 @@ +import os +import sqlite3 +import time +from typing import Callable, Optional +from functools import lru_cache +from utils.path_helpers import ensure_long_path + +verbose: bool = True + + +def _dlog(label: str, msg: str, cb: Optional[Callable[[str], None]] = None) -> None: + if not verbose: + return + ts = time.strftime("%H:%M:%S") + line = f"{ts} [{label}] {msg}" + if cb: + cb(line) + else: + print(line) + + +def _ensure_db(db_path: str) -> sqlite3.Connection: + os.makedirs(os.path.dirname(db_path), exist_ok=True) + conn = sqlite3.connect(db_path) + conn.execute( + """ + CREATE TABLE IF NOT EXISTS fingerprints ( + path TEXT PRIMARY KEY, + mtime REAL, + size INTEGER, + duration INT, + fingerprint TEXT + ); + """ + ) + + # --- Handle schema upgrades ------------------------------------------- + cols = { + row[1] for row in conn.execute("PRAGMA table_info(fingerprints)") + } + if "mtime" not in cols: + conn.execute("ALTER TABLE fingerprints ADD COLUMN mtime REAL") + if "size" not in cols: + conn.execute("ALTER TABLE fingerprints ADD COLUMN size INTEGER") + if "duration" not in cols: + conn.execute("ALTER TABLE fingerprints ADD COLUMN duration INT") + if "fingerprint" not in cols: + conn.execute("ALTER TABLE fingerprints ADD COLUMN fingerprint TEXT") + conn.commit() + + return conn + + +@lru_cache(maxsize=128) +def get_fingerprint( + path: str, + db_path: str, + compute_func: Callable[[str], tuple[int | None, str | None]], + log_callback: Optional[Callable[[str], None]] = None, +) -> Optional[str]: + """Return fingerprint for path using cache; compute if missing.""" + if log_callback is None: + log_callback = lambda msg: None + + path = ensure_long_path(path) + os.makedirs(os.path.dirname(db_path), exist_ok=True) + conn = _ensure_db(db_path) + try: + mtime = os.path.getmtime(path) + size = os.path.getsize(path) + except OSError as e: + log_callback(f"! Could not stat {path}: {e}") + conn.close() + return None + row = conn.execute( + "SELECT mtime, size, fingerprint FROM fingerprints WHERE path=?", + (path,), + ).fetchone() + cached_mtime = row[0] if row else None + cached_size = row[1] if row else None + if row and abs(cached_mtime - mtime) < 1e-6 and int(cached_size or 0) == int(size): + fp = row[2] + _dlog("FP", f"cache hit {path}", log_callback) + conn.close() + if isinstance(fp, (bytes, bytearray)): + try: + fp = fp.decode("utf-8") + except Exception: + fp = fp.decode("latin1", errors="ignore") + _dlog("FP", f"fingerprint={fp} prefix={fp[:16]}", log_callback) + return fp + if row: + _dlog( + "FP", + f"cache invalidated {path} stored_mtime={cached_mtime} stored_size={cached_size} new_mtime={mtime} new_size={size}", + log_callback, + ) + + _dlog("FP", f"cache miss {path}", log_callback) + duration, fp_hash = compute_func(path) + if fp_hash is not None: + _dlog("FP", f"computed fingerprint {fp_hash} prefix={fp_hash[:16]}", log_callback) + conn.execute( + "INSERT OR REPLACE INTO fingerprints (path, mtime, size, duration, fingerprint) VALUES (?, ?, ?, ?, ?)", + (path, mtime, size, duration, fp_hash), + ) + conn.commit() + conn.close() + return fp_hash + + +def flush_cache(db_path: str) -> None: + get_fingerprint.cache_clear() + if not os.path.exists(db_path): + return + try: + os.remove(db_path) + except Exception: + conn = sqlite3.connect(db_path) + conn.execute("DROP TABLE IF EXISTS fingerprints") + conn.commit() + conn.close() diff --git a/library_sync_indexer_engine/indexer_engine/fingerprint_generator.py b/library_sync_indexer_engine/indexer_engine/fingerprint_generator.py new file mode 100644 index 0000000..fb5d5ba --- /dev/null +++ b/library_sync_indexer_engine/indexer_engine/fingerprint_generator.py @@ -0,0 +1,224 @@ +import os +import sqlite3 +import tempfile +import threading +from typing import Callable, Iterable, List, Tuple +from pydub import AudioSegment, silence + + +SILENCE_THRESH = -50 # dBFS used for silence detection + + +def _with_thresh(func, *args, **kwargs): + try: + return func(*args, silence_threshold=SILENCE_THRESH, **kwargs) + except TypeError: + return func(*args, silence_thresh=SILENCE_THRESH, **kwargs) +from concurrent.futures import ProcessPoolExecutor +import acoustid +from utils.path_helpers import ensure_long_path + +SUPPORTED_EXTS = {".flac", ".m4a", ".aac", ".mp3", ".wav", ".ogg"} + + +def _trim_silence(path: str) -> str: + """Return path to temporary file with leading/trailing silence removed.""" + try: + ext = os.path.splitext(path)[1].lower().lstrip(".") or "wav" + audio = AudioSegment.from_file(path) + segs = _with_thresh(silence.detect_nonsilent, audio, min_silence_len=500) + if not segs: + return path + start = segs[0][0] + end = segs[-1][1] + trimmed = audio[start:end] + fd, tmp = tempfile.mkstemp(suffix=f".{ext}") + os.close(fd) + trimmed.export(tmp, format=ext) + return tmp + except Exception: + return path + + +def compute_fingerprint_for_file(args: Tuple[str, str, bool]) -> Tuple[str, int | None, str | None, str | None]: + """Compute fingerprint for a single file. Returns (path, duration, fp, error).""" + path, _db_path, trim = args + path = ensure_long_path(path) + tmp = None + try: + target = path + if trim: + tmp = _trim_silence(path) + target = tmp + duration, fp_hash = acoustid.fingerprint_file(target) + if tmp and tmp != path: + os.remove(tmp) + return path, duration, fp_hash, None + except Exception as e: # pragma: no cover - just to be safe + if tmp and tmp != path: + try: + os.remove(tmp) + except Exception: + pass + return path, None, None, str(e) + + +def _load_cached_entries(conn: sqlite3.Connection) -> dict[str, Tuple[float, int | None, str | None]]: + """Return a mapping of path -> (mtime, duration, fingerprint).""" + + cursor = conn.execute( + "SELECT path, mtime, duration, fingerprint FROM fingerprints" + ) + return {row[0]: (row[1], row[2], row[3]) for row in cursor.fetchall()} + + +def _gather_audio_files(root_path: str | Iterable[str]) -> List[str]: + """Return a list of audio files either from a directory walk or iterable.""" + + if isinstance(root_path, str): + audio_files: List[str] = [] + for dirpath, _, files in os.walk(root_path): + rel_dir = os.path.relpath(dirpath, root_path) + if {"not sorted", "playlists"} & {p.lower() for p in rel_dir.split(os.sep)}: + continue + for fname in files: + ext = os.path.splitext(fname)[1].lower() + if ext in SUPPORTED_EXTS: + audio_files.append(os.path.join(dirpath, fname)) + return audio_files + + return [p for p in root_path if os.path.splitext(p)[1].lower() in SUPPORTED_EXTS] + + +def compute_fingerprints_parallel( + root_path: str | Iterable[str], + db_path: str, + log_callback: Callable[[str], None] | None = None, + progress_callback: Callable[[int, int, str, str], None] | None = None, + max_workers: int | None = None, + trim_silence: bool = False, + phase: str = "A", + cancel_event: threading.Event | None = None, +) -> list[tuple[str, int | None, str | None]]: + """Compute fingerprints and return ``(path, duration, fingerprint)`` results. + + ``root_path`` may be a directory or an iterable of specific file paths. Existing + fingerprints in ``db_path`` are reused when their mtime matches, allowing + subsequent scans to skip unchanged files without sacrificing quality. + """ + if log_callback is None: + log_callback = print + + if progress_callback is None: + def progress_callback(current: int, total: int, msg: str, _phase: str) -> None: + pass + + if cancel_event is None: + cancel_event = threading.Event() + + if max_workers is None: + max_workers = os.cpu_count() or 1 + + db_folder = os.path.dirname(db_path) + os.makedirs(db_folder, exist_ok=True) + conn = sqlite3.connect(db_path) + conn.execute( + """ + CREATE TABLE IF NOT EXISTS fingerprints ( + path TEXT PRIMARY KEY, + mtime REAL, + duration INT, + fingerprint TEXT + ); + """ + ) + + results: list[tuple[str, int | None, str | None]] = [] + audio_files = _gather_audio_files(root_path) + + total = len(audio_files) + progress_callback(0, total, "Fingerprinting", phase) + + cached = _load_cached_entries(conn) + pending: list[str] = [] + completed = 0 + + for path in audio_files: + if cancel_event.is_set(): + log_callback("Cancellation requested before fingerprint scheduling") + break + mtime = os.path.getmtime(ensure_long_path(path)) + cache_entry = cached.get(path) + if cache_entry and cache_entry[2] is not None and cache_entry[0] == mtime: + duration, fp_hash = cache_entry[1], cache_entry[2] + results.append((path, duration, fp_hash)) + completed += 1 + progress_callback(completed, total, path, phase) + continue + pending.append(path) + + work_items = [(p, db_path, trim_silence) for p in pending] + + if pending: + with ProcessPoolExecutor(max_workers=max_workers) as exe: + futures = [exe.submit(compute_fingerprint_for_file, item) for item in work_items] + for fut in futures: + if cancel_event.is_set(): + cancelled = 0 + for remaining in futures: + if not remaining.done() and remaining.cancel(): + cancelled += 1 + log_callback(f"Cancellation requested; attempting to stop after current file ({cancelled} cancelled)") + break + + path, duration, fp_hash, err = fut.result() + completed += 1 + if err: + log_callback(f" ! Failed fingerprint {path}: {err}") + progress_callback(completed, total, path, phase) + if cancel_event.is_set(): + break + continue + mtime = os.path.getmtime(ensure_long_path(path)) + conn.execute( + "INSERT OR REPLACE INTO fingerprints (path, mtime, duration, fingerprint) VALUES (?, ?, ?, ?)", + (path, mtime, duration, fp_hash), + ) + results.append((path, duration, fp_hash)) + log_callback(f"Fingerprinted {path}") + progress_callback(completed, total, path, phase) + if completed % 50 == 0 or completed == total: + log_callback(f" • Fingerprinting {completed}/{total}") + if cancel_event.is_set(): + log_callback("Cancellation requested; stopping after current fingerprint") + break + else: + log_callback("All fingerprints loaded from cache") + + conn.commit() + conn.close() + return results + + + +def compute_fingerprints( + root_path: str, + db_path: str, + log_callback: Callable[[str], None] | None = None, + progress_callback: Callable[[int, int, str, str], None] | None = None, + max_workers: int | None = None, + trim_silence: bool = False, + phase: str = "A", + cancel_event: threading.Event | None = None, +) -> None: + """Backward-compatible wrapper around :func:`compute_fingerprints_parallel`.""" + return compute_fingerprints_parallel( + root_path, + db_path, + log_callback, + progress_callback, + max_workers, + trim_silence, + phase, + cancel_event, + ) diff --git a/library_sync_indexer_engine/indexer_engine/indexer_control.py b/library_sync_indexer_engine/indexer_engine/indexer_control.py new file mode 100644 index 0000000..d24a86e --- /dev/null +++ b/library_sync_indexer_engine/indexer_engine/indexer_control.py @@ -0,0 +1,13 @@ +import threading + +cancel_event = threading.Event() + +class IndexCancelled(Exception): + """Raised when indexing is cancelled.""" + pass + + +def check_cancelled() -> None: + """Raise IndexCancelled if the shared cancel_event is set.""" + if cancel_event.is_set(): + raise IndexCancelled() diff --git a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py new file mode 100644 index 0000000..5d962bb --- /dev/null +++ b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py @@ -0,0 +1,1174 @@ +# music_indexer_api.py + +import os +import re +import shutil # used for relocating special folders +import hashlib +import base64 +from collections import defaultdict +from typing import Dict, List +from dry_run_coordinator import DryRunCoordinator +from config import load_config + +def _verify_dependencies() -> None: + """Raise RuntimeError if the real mutagen library is missing.""" + try: + import mutagen # type: ignore + if getattr(mutagen.File, "__name__", "") == "": + raise ImportError + except Exception: # pragma: no cover - optional dependency + raise RuntimeError( + "Missing required library: mutagen. Install it via `pip install -r requirements.txt`." + ) +try: + from mutagen import File as MutagenFile + from mutagen.id3 import ID3NoHeaderError +except Exception: # pragma: no cover - optional dependency + def MutagenFile(*_a, **_k): + return None + + class ID3NoHeaderError(Exception): + """Fallback error when Mutagen is unavailable.""" + pass +from indexer_control import check_cancelled + +# ─── CONFIGURATION ───────────────────────────────────────────────────── +COMMON_ARTIST_THRESHOLD = 10 +REMIX_FOLDER_THRESHOLD = 3 +SUPPORTED_EXTS = {".flac", ".m4a", ".aac", ".mp3", ".wav", ".ogg"} +DEFAULT_FUZZY_FP_THRESHOLD = 0.1 # allowed fingerprint difference ratio +DEDUP_KEEP_DELTA_THRESHOLD = 50 # score difference to auto-delete duplicates +FINGERPRINT_TIMEOUT = 30 # seconds before fingerprint worker is timed out + +# ─── Helper: build primary_counts for the entire vault ───────────────────── +def build_primary_counts(root_path, progress_callback=None, phase="A"): + """ + Walk the entire vault (By Artist, By Year, Incoming, etc.) and + return a dict mapping each lowercase-normalized artist → total file count. + Falls back to filename if metadata “artist” is missing. + + ``progress_callback`` will be invoked every 100 files with + ``(count, 0, path)`` allowing callers to show progress without + a separate pre-scan. + """ + if progress_callback is None: + def progress_callback(_c, _t, _m, _p): + pass + + counts = {} + idx = 0 + for dirpath, _, files in os.walk(root_path): + rel_dir = os.path.relpath(dirpath, root_path) + if {"not sorted", "playlists"} & {p.lower() for p in rel_dir.split(os.sep)}: + continue + for fname in files: + ext = os.path.splitext(fname)[1].lower() + if ext not in SUPPORTED_EXTS: + continue + + path = os.path.join(dirpath, fname) + tags = get_tags(path) + + idx += 1 + if idx % 100 == 0: + progress_callback(idx, 0, os.path.relpath(path, root_path), phase) + + # Normalize artist identically to Phase 4 + raw = (tags.get("artist") or "").strip() + + if not raw: + name_only = os.path.splitext(fname)[0] + if "_" in name_only: + raw = name_only.split("_", 1)[0] + elif " - " in name_only: + raw = name_only.split(" - ", 1)[0] + else: + raw = name_only + + primary, _ = extract_primary_and_collabs(raw) + p_lower = primary.lower() + counts[p_lower] = counts.get(p_lower, 0) + 1 + + return counts + +# ─── Shared Utility Functions ───────────────────────────────────────── +def sanitize(name: str) -> str: + invalid = r'<>:"/\\|?*' + return "".join(c for c in (name or "Unknown") if c not in invalid).strip() or "Unknown" + +def collapse_repeats(name: str) -> str: + """ + If name is something like 'DROELOEDROELOE', collapse to 'DROELOE'. + For strings shorter than 4 characters, return unchanged. + """ + if not name or len(name) < 4: + return name or "Unknown" + for length in range(1, len(name)//2 + 1): + if len(name) % length == 0: + chunk = name[:length] + if chunk * (len(name)//length) == name: + return chunk + return name + +def is_repeated(name: str) -> bool: + """ + Detect if the entire artist tag is a repeated substring, + e.g. 'DROELOEDROELOE'. For strings shorter than 4, return False. + """ + if not name or len(name) < 4: + return False + for length in range(1, len(name)//2 + 1): + if len(name) % length == 0: + chunk = name[:length] + if chunk * (len(name)//length) == name: + return True + return False + +def get_tags(path: str): + """ + Read basic tags using Mutagen (artist, title, album, year, track, genre). + Return a dict with those fields (or None if missing). + """ + try: + audio = MutagenFile(path, easy=True) + if not audio or not audio.tags: + return {"artist": None, "title": None, "album": None, + "year": None, "track": None, "genre": None} + tags = audio.tags + artist = tags.get("artist", [None])[0] + title = tags.get("title", [None])[0] + album = tags.get("album", [None])[0] + raw_date = tags.get("date", [None])[0] or tags.get("year", [None])[0] + year = raw_date[:4] if raw_date else None + raw_track = tags.get("tracknumber", [None])[0] or tags.get("track", [None])[0] + track = None + if raw_track: + try: + track = int(raw_track.split("/")[0]) + except Exception: + track = None + genre = tags.get("genre", [None])[0] + return {"artist": artist, "title": title, "album": album, + "year": year, "track": track, "genre": genre} + except Exception: + return {"artist": None, "title": None, "album": None, + "year": None, "track": None, "genre": None} + +def extract_primary_and_collabs(raw_artist: str): + """ + Use collapse_repeats on raw_artist. Then: + - If raw_artist contains "/" → split on "/" → first part is primary, rest are collabs. + - Else look for separators: " feat.", " ft.", " & ", " x ", ", ", ";" + First segment is primary, remainder split by commas/& is collabs. + - Else split on uppercase boundary, treat first as primary and rest as a single collab string. + - Otherwise, primary = raw_artist, collabs = []. + """ + if not raw_artist: + return ("Unknown", []) + text = raw_artist.strip() + text = collapse_repeats(text) + + if "/" in text: + parts = [collapse_repeats(p.strip()) for p in text.split("/") if p.strip()] + return (parts[0], parts[1:]) + + lowered = text.lower() + separators = [" feat.", " ft.", " & ", " x ", ", ", ";"] + for sep in separators: + idx = lowered.find(sep) + if idx != -1: + primary = collapse_repeats(text[:idx].strip()) + rest = text[idx + len(sep):].strip() + subparts = [collapse_repeats(p.strip()) for p in re.split(r"\s*&\s*|,\s*", rest) if p.strip()] + return (primary, subparts) + + parts = re.split(r'(?<=[a-z])(?=[A-Z])', text) + if len(parts) > 1: + primary = collapse_repeats(parts[0].strip()) + collabs = [" & ".join(collapse_repeats(p.strip()) for p in parts[1:])] + return (primary, collabs) + + return (text, []) + +def derive_tags_from_path(filepath: str, music_root: str): + """ + If a file lives in subfolders under music_root, gather those folder names + as “inherited tags.” E.g. /…/By Artist/ArtistName/AlbumX/Track.mp3 → {"ArtistName", "AlbumX"}. + """ + rel = os.path.relpath(os.path.dirname(filepath), music_root) + parts = [p for p in rel.split(os.sep) if p and p != "."] + return set(parts) + + +def _parse_fp(fp: str) -> tuple[str, list[int] | bytes] | None: + """Parse fingerprint string into integer list or decoded bytes.""" + text = fp.strip() + if not text: + return None + try: + arr = [int(x) for x in text.replace(',', ' ').split()] + if arr: + return ("ints", arr) + except Exception: + pass + try: + data = base64.urlsafe_b64decode(text + '=' * (-len(text) % 4)) + if data: + return ("bytes", data) + except Exception: + pass + return None + + +def _hamming_ints(a: list[int], b: list[int]) -> float: + n = min(len(a), len(b)) + if n == 0: + return 1.0 + diff = sum(x != y for x, y in zip(a[:n], b[:n])) + return diff / n + + +def _hamming_bytes(a: bytes, b: bytes) -> float: + n = min(len(a), len(b)) + if n == 0: + return 1.0 + diff = 0 + for i in range(n): + diff += (a[i] ^ b[i]).bit_count() + return diff / (n * 8) + + +def fingerprint_distance(fp1: str | None, fp2: str | None) -> float: + """Return normalized Hamming distance between two fingerprint strings.""" + if not fp1 or not fp2: + return 1.0 + if fp1 == fp2: + return 0.0 + p1 = _parse_fp(fp1) + p2 = _parse_fp(fp2) + if not p1 or not p2 or p1[0] != p2[0]: + return 1.0 + kind, a = p1 + _, b = p2 + if kind == "ints": + return _hamming_ints(a, b) # type: ignore[arg-type] + else: + return _hamming_bytes(a, b) # type: ignore[arg-type] + + +def _keep_score(path: str, info: dict, ext_priority: dict) -> float: + """Compute keep score for a file based on extension, metadata and filename.""" + ext = os.path.splitext(path)[1].lower() + pri = ext_priority.get(ext, 99) + ext_score = 1000.0 / (pri + 1) + meta_score = info.get("meta_count", 0) * 10 + fname_score = len(os.path.splitext(os.path.basename(path))[0]) + return ext_score + meta_score + fname_score + + + + +# ─── A. HELPER: COMPUTE MOVES & TAG INDEX ─────────────────────────────── + +def compute_moves_and_tag_index( + root_path, + log_callback=None, + progress_callback=None, + dry_run=False, + enable_phase_c=False, + flush_cache=False, + max_workers=None, + coord=None, +): + """ + 1) Determine MUSIC_ROOT: if root_path/Music exists, use that; otherwise root_path itself. + 2) Scan for all audio files under MUSIC_ROOT. + 3) Deduplicate tracks by fingerprint only and delete lower-priority duplicates. + If ``dry_run`` is True, deduplication actions are simulated only and no + files are modified. + 4) Read metadata into `songs` dict, build: + - album_counts: how many genuine (non-remix) tracks per (artist, album) + - remix_counts: how many remix‐tagged tracks per (artist, album) + - cover_counts: how many files share each embedded cover + 5) Phase 4: For each entry in `songs`, decide new folder & filename, + with special‐case for small “(Remixes)” albums. + Returns: + - moves: { old_path: new_path, ... } + - tag_index: { new_path: { "leftover_tags": [...], "old_paths": [...] }, ... } + - decision_log: list of strings explaining each track’s decision + + ``coord`` can be provided to collect additional diagnostic data during + near-duplicate detection. + """ + _verify_dependencies() + if log_callback is None: + def log_callback(msg): + pass + if progress_callback is None: + def progress_callback(current, total, message, phase): + pass + + cfg = load_config() + fuzzy_fp_threshold = float(cfg.get("fuzzy_fp_threshold", DEFAULT_FUZZY_FP_THRESHOLD)) + check_cancelled() + + # ─── 1) Determine MUSIC_ROOT ────────────────────────────────────────────── + MUSIC_ROOT = os.path.join(root_path, "Music") \ + if os.path.isdir(os.path.join(root_path, "Music")) \ + else root_path + + SUPPORTED_EXTS = {".flac", ".m4a", ".aac", ".mp3", ".wav", ".ogg"} + + docs_dir = os.path.join(root_path, "Docs") + os.makedirs(docs_dir, exist_ok=True) + db_path = os.path.join(docs_dir, ".soundvault.db") + if flush_cache: + from fingerprint_cache import flush_cache as _flush + _flush(db_path) + + # --- Phase 0: Pre-scan entire vault (under MUSIC_ROOT) --- + global_counts = build_primary_counts(MUSIC_ROOT, progress_callback, phase="A") + log_callback(f" → Pre-scan: found {len(global_counts)} unique artists") + log_callback(f" → DEBUG: MUSIC_ROOT = {MUSIC_ROOT}") + log_callback(f" → DEBUG: droeloe count = {global_counts.get('droeloe', 0)}") + + # --- Phase 1: Scan for audio files --- + log_callback("1/6: Discovering audio files…") + all_audio = [] + idx = 0 + for dirpath, _, files in os.walk(MUSIC_ROOT): + rel_dir = os.path.relpath(dirpath, root_path) + if {"not sorted", "playlists"} & {p.lower() for p in rel_dir.split(os.sep)}: + continue + for fname in files: + ext = os.path.splitext(fname)[1].lower() + if ext in SUPPORTED_EXTS: + full = os.path.join(dirpath, fname) + if {"not sorted", "playlists"} & {p.lower() for p in os.path.relpath(full, root_path).split(os.sep)}: + continue + all_audio.append(full) + idx += 1 + progress_callback(idx, 0, f"Scanning {os.path.relpath(full, root_path)}", "A") + total_audio = idx + log_callback(f" → Found {total_audio} audio files.") + + # ─── Phase 2: Deduplicate using fingerprints ───────────────────────────── + log_callback("2/6: Deduplicating by fingerprint…") + EXT_PRIORITY = {".flac": 0, ".m4a": 1, ".aac": 1, ".mp3": 2, ".wav": 3, ".ogg": 4} + + path_fps: Dict[str, str | None] = {} + from fingerprint_cache import get_fingerprint + + def _compute(path: str) -> tuple[int | None, str | None]: + try: + import acoustid + return acoustid.fingerprint_file(path) + except Exception: + return None, None + + progress_callback(0, len(all_audio), "Fingerprinting", "A") + for idx, p in enumerate(all_audio, start=1): + fp_val = get_fingerprint(p, db_path, _compute) + path_fps[p] = fp_val + progress_callback(idx, len(all_audio), os.path.relpath(p, root_path), "A") + if idx % 50 == 0 or idx == len(all_audio): + log_callback(f" • Fingerprinting {idx}/{len(all_audio)}") + + file_infos: Dict[str, Dict[str, str | None]] = {} + for idx, fullpath in enumerate(all_audio, start=1): + if idx % 50 == 0 or idx == total_audio: + log_callback(f" • Processing file {idx}/{total_audio} for dedupe") + data = get_tags(fullpath) + raw_artist = data["artist"] or os.path.splitext(os.path.basename(fullpath))[0] + raw_artist = collapse_repeats(raw_artist) + title = data["title"] or os.path.splitext(os.path.basename(fullpath))[0] + album = data["album"] or "" + meta_count = sum( + 1 + for t in [data.get("artist"), data.get("title"), data.get("album"), data.get("track"), data.get("year"), data.get("genre")] + if t + ) + primary, _ = extract_primary_and_collabs(raw_artist) + fp = path_fps.get(fullpath) + if isinstance(fp, (bytes, bytearray)): + try: + fp = fp.decode("utf-8") + except Exception: + fp = fp.decode("latin1", errors="ignore") + file_infos[fullpath] = { + "primary": primary.lower(), + "title": title.lower(), + "album": album.lower(), + "fp": fp, + "meta_count": meta_count, + } + + to_delete: Dict[str, str] = {} + fp_groups: Dict[str, List[str]] = defaultdict(list) + for path, info in file_infos.items(): + if info["fp"]: + fp_groups[info["fp"]].append(path) + + kept_files = set(file_infos.keys()) + review_root = os.path.join(root_path, "Manual Review") + os.makedirs(review_root, exist_ok=True) + + for fp, paths in fp_groups.items(): + if len(paths) <= 1: + continue + scored = sorted(paths, key=lambda p: _keep_score(p, file_infos[p], EXT_PRIORITY), reverse=True) + if len(scored) < 2: + continue + delta = _keep_score(scored[0], file_infos[scored[0]], EXT_PRIORITY) - _keep_score(scored[1], file_infos[scored[1]], EXT_PRIORITY) + if delta >= DEDUP_KEEP_DELTA_THRESHOLD: + for loser in scored[1:]: + if loser not in to_delete: + to_delete[loser] = "Exact FP match" + kept_files.discard(loser) + else: + meta_counts = {p: file_infos[p].get("meta_count", 0) for p in scored} + max_meta = max(meta_counts.values()) + winner = None + for p in scored: + if meta_counts[p] == max_meta: + winner = p + break + for p in scored: + if p == winner: + continue + if p not in to_delete: + to_delete[p] = "Exact FP match" + kept_files.discard(p) + log_callback( + f"Ambiguous duplicates for fingerprint {fp} auto-resolved—kept {os.path.basename(winner)}." + ) + + if coord is not None: + coord.add_exact_dupes([p for p, r in to_delete.items() if r == "Exact FP match"]) + + + # --- Near-duplicate detection ------------------------------------------------- + from near_duplicate_detector import find_near_duplicates + log_callback(" • Detecting near-duplicates…") + near_dupes = find_near_duplicates( + {p: file_infos[p] for p in kept_files}, + EXT_PRIORITY, + fuzzy_fp_threshold, + log_callback, + enable_phase_c, + coord, + max_workers, + ) + for loser, reason in near_dupes.items(): + if loser not in to_delete: + to_delete[loser] = reason + kept_files.discard(loser) + + + trash_dir = os.path.join(root_path, "Trash") + os.makedirs(trash_dir, exist_ok=True) + for loser, reason in to_delete.items(): + if dry_run: + log_callback( + f" ? (dry-run) Would move duplicate {loser} to Trash ({reason})" + ) + else: + try: + dest = os.path.join(trash_dir, os.path.basename(loser)) + base, ext = os.path.splitext(os.path.basename(loser)) + counter = 1 + while os.path.exists(dest): + dest = os.path.join(trash_dir, f"{base}_{counter}{ext}") + counter += 1 + shutil.move(loser, dest) + log_callback( + f" - Moved duplicate to Trash ({reason}): {loser}" + ) + except Exception as e: + log_callback(f" ! Failed to move {loser} to Trash: {e}") + + # ─── Phase 3: Read metadata & build counters ───────────────────────────────── + log_callback("3/6: Reading metadata and building counters…") + songs = {} + album_counts = defaultdict(int) + remix_counts = defaultdict(int) + cover_counts = defaultdict(int) + total_kept = len(kept_files) + progress_callback(0, total_kept, "Reading metadata", "B") + + for idx, fullpath in enumerate(kept_files, start=1): + if idx % 50 == 0 or idx == total_kept: + log_callback(f" • Reading metadata {idx}/{total_kept}") + progress_callback(idx, total_kept, f"Metadata {os.path.relpath(fullpath, root_path)}", "B") + + data = get_tags(fullpath) + raw_artist = data["artist"] or os.path.splitext(os.path.basename(fullpath))[0] + raw_artist = collapse_repeats(raw_artist) + title = data["title"] or os.path.splitext(os.path.basename(fullpath))[0] + album = data["album"] + year = data["year"] + genre = data["genre"] + track = data["track"] + + # 1) Primary & collabs (preserve original case) + primary, collabs = extract_primary_and_collabs(raw_artist) + p_lower = primary.lower() + + # 2) album_counts for genuine (non-remix) tracks under each (artist, album) + if album and "remix" not in title.lower() and album.strip().lower() != title.strip().lower(): + album_counts[(p_lower, album.lower())] += 1 + + # 3) remix_counts for tracks whose album tag contains "remix" + if album and "remix" in album.lower(): + remix_counts[(p_lower, album.lower())] += 1 + + # 4) EXTRACT EMBEDDED COVER DATA → compute SHA-1 and store first 10 hex digits + cover_hash = None + try: + audio_file = MutagenFile(fullpath) + img_data = None + if hasattr(audio_file, "tags") and audio_file.tags is not None: + for key in audio_file.tags.keys(): + if key.startswith("APIC"): + img_data = audio_file.tags[key].data + break + if img_data is None and audio_file.__class__.__name__ == "FLAC": + pics = audio_file.pictures + if pics: + img_data = pics[0].data + if img_data: + sha1 = hashlib.sha1(img_data).hexdigest() + cover_hash = sha1[:10] + cover_counts[cover_hash] += 1 + except ID3NoHeaderError: + pass + except Exception: + pass + + folder_tags = derive_tags_from_path(fullpath, MUSIC_ROOT) + + songs[fullpath] = { + "raw_artist": raw_artist, + "primary": primary, + "collabs": collabs, + "title": title, + "album": album, + "year": year, + "genre": genre, + "track": track, + "cover_hash": cover_hash, + "folder_tags": folder_tags, + "missing_core": not data.get("artist") or not data.get("title"), + } + + log_callback(f" → Collected metadata for {total_kept} files.") + + # ─── Phase 4: Determine destination for each file (with logging) ───────────── + log_callback("4/6: Determining destination paths for each file…") + tag_index = {} + moves = {} + decision_log = [] + index = 0 + total = len(songs) + + # Precompute any necessary lookups from songs data + # (album_counts, remix_counts, etc. were built in Phase 3) + + for old_path, info in songs.items(): + index += 1 + if index % 50 == 0 or index == total: + log_callback(f" • Determining destination {index}/{total}") + + raw_artist = info["raw_artist"] + title = info["title"] or "" + album = info["album"] or "" + year = info["year"] or "Unknown" + track = info["track"] + folders = info["folder_tags"] + if info.get("missing_core"): + candidate = os.path.join(review_root, os.path.basename(old_path)) + root_c, ext_c = os.path.splitext(candidate) + idx_dup = 1 + while os.path.exists(candidate) or candidate in moves.values(): + candidate = f"{root_c} ({idx_dup}){ext_c}" + idx_dup += 1 + moves[old_path] = candidate + decision_log.append( + f" → Missing metadata, placed under Manual Review/{os.path.basename(candidate)}" + ) + continue + + # Normalize again for consistent global lookup + primary_norm, _ = extract_primary_and_collabs(raw_artist) + p_lower = primary_norm.lower() + + # Lookup how many total tracks this artist has across the entire vault + count_now = global_counts.get(p_lower, 0) + decision_log.append(f"Song: {raw_artist} – {title} ({album}) → global_counts[{raw_artist}] = {count_now}") + + # ─── 4.1) EARLY BAIL-OUT: “Rare” artists go straight to By Year ───────────────── + if count_now < COMMON_ARTIST_THRESHOLD: + base_folder = os.path.join(MUSIC_ROOT, "By Year", sanitize(year)) + decision_log.append( + f" Early-exit: Count {count_now} < {COMMON_ARTIST_THRESHOLD} → group under By Year/{year}" + ) + + # Build the new filename as usual (rename invalid artists or use sanitized tags) + if "/" in raw_artist or is_repeated(raw_artist): + new_filename = os.path.basename(old_path) + decision_log.append(f" Raw artist '{raw_artist}' malformed → keeping filename '{new_filename}'") + else: + ext = os.path.splitext(old_path)[1].lower() + filename_artist = sanitize(raw_artist) + track_str = f"{track:02d}" if track is not None else "00" + title_str = sanitize(title) + new_filename = f"{filename_artist}_{track_str}_{title_str}{ext}" + decision_log.append(f" Renaming to '{new_filename}' (using raw artist)") + + new_path = os.path.join(base_folder, new_filename) + moves[old_path] = new_path + decision_log.append( + f" → Final: '{os.path.relpath(new_path, MUSIC_ROOT)}'" + ) + # Skip all other logic (no remix or album/singles checks) + continue + + # At this point, count_now ≥ COMMON_ARTIST_THRESHOLD + # ─── 4.2) SPECIAL CASE: Album ends with “(Remixes)” ────────────────────────── + if album and album.strip().lower().endswith("(remixes)"): + rcount = remix_counts.get((p_lower, album.lower()), 0) + decision_log.append( + f" Album '{album}' has {rcount} remix-tagged tracks (threshold={REMIX_FOLDER_THRESHOLD})" + ) + + # Only force into By Artist if both remix-count AND artist-count thresholds are met + if rcount >= REMIX_FOLDER_THRESHOLD and count_now >= COMMON_ARTIST_THRESHOLD: + # Promote primary to remixer (first part of raw_artist before “/”) + main_artist = raw_artist.split("/", 1)[0] + p_lower = main_artist.lower() + decision_log.append( + f" → Enough remixes ({rcount} ≥ {REMIX_FOLDER_THRESHOLD}) " + f"AND count_now ({count_now}) ≥ {COMMON_ARTIST_THRESHOLD}; force primary = '{main_artist}'" + ) + artist_folder = os.path.join(MUSIC_ROOT, "By Artist", sanitize(main_artist)) + base_folder = os.path.join(artist_folder, sanitize(album)) + + # Build filename (skip normal album logic) + basename = os.path.basename(old_path) + if "/" in raw_artist or is_repeated(raw_artist): + new_filename = basename + decision_log.append(f" Raw artist '{raw_artist}' malformed → keeping '{basename}'") + else: + ext = os.path.splitext(old_path)[1].lower() + filename_artist = sanitize(raw_artist) + track_str = f"{track:02d}" if track is not None else "00" + title_str = sanitize(title) + new_filename = f"{filename_artist}_{track_str}_{title_str}{ext}" + decision_log.append(f" Renaming to '{new_filename}' (using raw artist)") + + new_path = os.path.join(base_folder, new_filename) + moves[old_path] = new_path + decision_log.append( + f" → (Remixes folder) placed under By Artist/{main_artist}/{album} " + f"→ Final: '{os.path.relpath(new_path, MUSIC_ROOT)}'" + ) + continue + else: + decision_log.append( + f" → Either only {rcount} remixes (< {REMIX_FOLDER_THRESHOLD}) " + f"or count_now ({count_now}) < {COMMON_ARTIST_THRESHOLD}; " + f"fall back to album/singles logic" + ) + + # ─── 4.3) BROAD COLLABORATOR RANKING (unchanged) ──────────────────────────── + all_candidates = [info["primary"]] + info["collabs"] + counts = {artist: global_counts.get(artist.lower(), 0) for artist in all_candidates} + best_artist = max(counts, key=lambda a: counts[a]) + decision_log.append(f" Candidates: {all_candidates}, counts: {counts}") + primary = best_artist + p_lower = primary.lower() + decision_log.append(f" → After ranking, primary = '{primary}'") + + # Build “By Artist/” base path + artist_folder = os.path.join(MUSIC_ROOT, "By Artist", sanitize(primary)) + + # ─── 4.4) COMMON ARTIST: decide between “Album” vs. “Singles” ─────────────── + # (At this point, count_now ≥ COMMON_ARTIST_THRESHOLD is guaranteed.) + if not album or album.strip().lower() == title.strip().lower(): + # No album tag (or album == title) → put into “ - Singles” + base_folder = os.path.join(artist_folder, f"{sanitize(primary)} - Singles") + decision_log.append( + f" Count {count_now} ≥ {COMMON_ARTIST_THRESHOLD}, no album or album=title " + f"→ group under '{primary} - Singles'" + ) + else: + c2 = album_counts.get((p_lower, album.lower()), 0) + if c2 > 3: + # Enough tracks in this album → put into “/” + base_folder = os.path.join(artist_folder, sanitize(album)) + decision_log.append( + f" Count {count_now} ≥ {COMMON_ARTIST_THRESHOLD}, album_count={c2} > 3 " + f"→ group under Album '{album}'" + ) + else: + # Few tracks in album → treat as a “Singles” release + base_folder = os.path.join(artist_folder, f"{sanitize(primary)} - Singles") + decision_log.append( + f" Count {count_now} ≥ {COMMON_ARTIST_THRESHOLD}, but " + f"album_count={c2} ≤ 3 → group under '{primary} - Singles'" + ) + + # Build filename for all “common” tracks + basename = os.path.basename(old_path) + if "/" in raw_artist or is_repeated(raw_artist): + new_filename = basename + decision_log.append(f" Raw artist '{raw_artist}' malformed → keeping '{basename}'") + else: + ext = os.path.splitext(old_path)[1].lower() + filename_artist = sanitize(raw_artist) + track_str = f"{track:02d}" if track is not None else "00" + title_str = sanitize(title) + new_filename = f"{filename_artist}_{track_str}_{title_str}{ext}" + decision_log.append(f" Renaming to '{new_filename}' (using raw artist)") + + new_path = os.path.join(base_folder, new_filename) + moves[old_path] = new_path + decision_log.append( + f" → Final: '{os.path.relpath(new_path, MUSIC_ROOT)}'" + ) + + # ─── 4.4) Build “leftover_tags” for HTML preview (optional) ─────────────── + leftover = set(folders) + leftover.discard(primary) + if album: + leftover.discard(album) + leftover.discard(year) + if genre: + leftover.add(genre) + + tag_index[new_path] = { + "leftover_tags": sorted(leftover), + "old_paths": sorted(folders) + } + + log_callback(" → Destination paths determined for all files.") + return moves, tag_index, decision_log + + +# ─── B. BUILD DRY-RUN HTML ───────────────────────────────────────────── + +def build_dry_run_html( + root_path, + output_html_path, + log_callback=None, + enable_phase_c=False, + flush_cache=False, + max_workers=None, +): + """ + 1) Call compute_moves_and_tag_index() to get (moves, tag_index, decision_log). + 2) Write a dry-run HTML to output_html_path showing the proposed tree. + Does NOT move any files. + """ + if log_callback is None: + def log_callback(msg): pass + + coord = DryRunCoordinator() + moves, tag_index, _ = compute_moves_and_tag_index( + root_path, + log_callback, + None, + dry_run=True, + enable_phase_c=enable_phase_c, + flush_cache=flush_cache, + max_workers=max_workers, + coord=coord, + ) + + log_callback("5/6: Writing dry-run HTML…") + + def build_exact_metadata_section() -> str: + lines = ["

      Phase A – Exact Metadata

      ", "
      "]
      +        lines.append(f"{sanitize(os.path.basename(root_path))}/")
      +        tree_nodes = set()
      +        for new_path in moves.values():
      +            parts = os.path.relpath(new_path, root_path).split(os.sep)
      +            for i in range(1, len(parts) + 1):
      +                subtree = os.path.join(root_path, *parts[:i])
      +                tree_nodes.add(subtree)
      +        for node in sorted(tree_nodes, key=lambda p: os.path.relpath(p, root_path)):
      +            rel = os.path.relpath(node, root_path)
      +            depth = rel.count(os.sep)
      +            indent = "    " * depth
      +            _, ext = os.path.splitext(node)
      +            if os.path.isdir(node) or ext.lower() not in {".flac", ".m4a", ".aac", ".mp3", ".wav", ".ogg"}:
      +                lines.append(f"{indent}{sanitize(os.path.basename(node))}/")
      +            else:
      +                fname = os.path.basename(node)
      +                leftover = tag_index.get(node, {}).get("leftover_tags", [])
      +                tags_str = ", ".join(leftover) if leftover else ""
      +                line = f"{indent}- {sanitize(fname)}"
      +                if tags_str:
      +                    line += f"  [{sanitize(tags_str)}]"
      +                lines.append(line)
      +        lines.append("
      ") + return "\n".join(lines) + + sec_a = build_exact_metadata_section() + coord.set_html_section('A', sec_a) + + html_body = coord.assemble_final_report() + + with open(output_html_path, "w", encoding="utf-8") as out: + out.write( + "\n\n\n \n " + f"Music Index (Dry Run) – {sanitize(os.path.basename(root_path))}\n " + "\n\n\n" + ) + out.write(html_body) + if html_body and not html_body.endswith("\n"): + out.write("\n") + out.write("\n\n") + + log_callback(f"✓ Dry-run HTML written to: {output_html_path}") + return {"moved": 0, "html": output_html_path, "dry_run": True} + + +# ─── C. APPLY MOVES ───────────────────────────────────────────────────── + +def apply_indexer_moves( + root_path, + log_callback=None, + progress_callback=None, + create_playlists: bool = True, +): + """ + 1) Call compute_moves_and_tag_index() to get (moves, tag_index, decision_log). + 2) Move/rename each file in `moves`. + 3) Move any leftover non-audio or album cover images into Trash or into the correct folder. + 4) Remove empty directories in two passes: first those that held moved + files, then a library-wide sweep for any remaining empty folders. + Returns summary: {"moved": , "errors": []}. + Set ``create_playlists`` to ``False`` to skip playlist generation at the end. + """ + if log_callback is None: + def log_callback(msg): + pass + if progress_callback is None: + def progress_callback(current, total, path, phase): + pass + + cfg = load_config() + trim_silence = bool(cfg.get("trim_silence", False)) + + # Ensure Not Sorted directory exists for user exclusions + not_sorted_dir = os.path.join(root_path, "Not Sorted") + os.makedirs(not_sorted_dir, exist_ok=True) + + # Determine where your actual music lives + music_root = os.path.join(root_path, "Music") + if not os.path.isdir(music_root): + music_root = root_path + + + # ─── Phase 0.5: Compute and cache fingerprints ───────────────── + docs_dir = os.path.join(root_path, "Docs") + os.makedirs(docs_dir, exist_ok=True) + db_path = os.path.join(docs_dir, ".soundvault.db") + from fingerprint_generator import compute_fingerprints_parallel + compute_fingerprints_parallel( + root_path, + db_path, + log_callback, + progress_callback, + None, + trim_silence, + phase="A", + ) + + moves, _, _ = compute_moves_and_tag_index( + root_path, + log_callback, + progress_callback, + dry_run=False, + enable_phase_c=False, + flush_cache=False, + max_workers=None, + coord=None, + ) + + + + total_moves = len(moves) + + MUSIC_ROOT = os.path.join(root_path, "Music") \ + if os.path.isdir(os.path.join(root_path, "Music")) \ + else root_path + SUPPORTED_EXTS = {".flac", ".m4a", ".aac", ".mp3", ".wav", ".ogg"} + IMAGE_EXTS = {".jpg", ".jpeg", ".png", ".gif"} + + summary = {"moved": 0, "errors": []} + + # Build a mapping: old_dir → set of target_dirs for audio files in that folder + olddir_to_newdirs = defaultdict(set) + for old_path, new_path in moves.items(): + old_dir = os.path.dirname(old_path) + new_dir = os.path.dirname(new_path) + olddir_to_newdirs[old_dir].add(new_dir) + + # Phase 5: Move audio files + for idx, (old_path, new_path) in enumerate(moves.items(), start=1): + if progress_callback: + progress_callback(idx, total_moves, old_path, "C") + if idx % 50 == 0 or idx == total_moves: + log_callback(f" • Moving file {idx}/{total_moves}") + os.makedirs(os.path.dirname(new_path), exist_ok=True) + try: + if os.path.abspath(old_path) != os.path.abspath(new_path): + shutil.move(old_path, new_path) + summary["moved"] += 1 + except Exception as e: + err = f"Failed to move {old_path} → {new_path}: {e}" + summary["errors"].append(err) + log_callback(f" ! {err}") + + # Phase 6: Handle non-audio leftovers… + docs_dir = os.path.join(root_path, "Docs") + trash_dir = os.path.join(root_path, "Trash") + os.makedirs(docs_dir, exist_ok=True) + os.makedirs(trash_dir, exist_ok=True) + + for dirpath, dirnames, filenames in os.walk(MUSIC_ROOT, topdown=False): + # 1) Don’t recurse back into Trash/, Docs/, Playlists, or Not Sorted/ + dirnames[:] = [d for d in dirnames if d.lower() not in ("trash", "docs", "not sorted", "playlists")] + if {"not sorted", "playlists"} & {p.lower() for p in os.path.relpath(dirpath, root_path).split(os.sep)}: + continue + + for fname in filenames: + full = os.path.join(dirpath, fname) + ext = os.path.splitext(fname)[1].lower() + + # 2) Skip files already moved or valid audio + if full in moves or ext in SUPPORTED_EXTS: + continue + + target = docs_dir if ext in (".txt", ".html", ".db") else trash_dir + if target is docs_dir: + log_callback(f"Moving doc to Docs/: {full}") + else: + log_callback(f"Moving to Trash/: {full}") + try: + shutil.move(full, os.path.join(target, fname)) + except Exception as e: + log_callback(f" ! Failed to move leftover {full}: {e}") + + # Phase 6b: Remove empty directories that held moved files + skip_names = {"trash", "docs", "not sorted", "playlists"} + touched_dirs = sorted(olddir_to_newdirs.keys(), key=lambda p: p.count(os.sep), reverse=True) + for base in touched_dirs: + if not os.path.isdir(base): + continue + for dirpath, dirnames, filenames in os.walk(base, topdown=False): + rel = os.path.relpath(dirpath, root_path) + first = rel.split(os.sep, 1)[0].lower() if rel != "." else "" + if first in skip_names: + continue + if os.path.normpath(dirpath) in (root_path, MUSIC_ROOT): + continue + if not os.listdir(dirpath): + try: + os.rmdir(dirpath) + log_callback(f"Removed empty folder: {dirpath}") + except OSError: + pass + + # Phase 6c: Second pass to remove any remaining empty directories + for dirpath, dirnames, filenames in os.walk(root_path, topdown=False): + dirnames[:] = [d for d in dirnames if d.lower() not in skip_names] + if os.path.normpath(dirpath) in (root_path, MUSIC_ROOT): + continue + rel = os.path.relpath(dirpath, root_path) + first = rel.split(os.sep, 1)[0].lower() if rel != "." else "" + if first in skip_names: + continue + if not os.listdir(dirpath): + try: + os.rmdir(dirpath) + log_callback(f"Removed empty folder: {dirpath}") + except OSError: + pass + + if create_playlists: + # ─── Phase 7: Generate Playlists with edge-case handling ──────────── + try: + from playlist_generator import generate_playlists + log_callback("7/7: Generating safe playlists…") + + def plog(msg): + log_callback(msg) + if progress_callback and msg.startswith("→ Writing playlist:"): + playlist_file = msg.split(":", 1)[1].strip() + progress_callback(total_moves, total_moves, playlist_file, "C") + + generate_playlists( + moves, + root_path, + output_dir=None, # default: root_path/Playlists + valid_exts=None, # default extensions + overwrite=True, + log_callback=plog, + ) + log_callback("✓ Robust playlists created.") + except ImportError: + log_callback("! playlist_generator.py missing; skipping playlists.") + except Exception as e: + log_callback(f"! Playlist generation error: {e}") + else: + log_callback("• Playlist creation disabled") + + return summary + + +# ─── D. HIGH-LEVEL “RUN FULL INDEXER” ─────────────────────────────────── + +def run_full_indexer( + root_path, + output_html_path, + dry_run_only: bool = False, + log_callback=None, + progress_callback=None, + enable_phase_c: bool = False, + flush_cache: bool = False, + max_workers: int | None = None, + create_playlists: bool = True, +): + """ + 1) Call compute_moves_and_tag_index() to get (moves, tag_index, decision_log). + 2) Write a detailed log file `indexer_log.txt` under root_path. + 3) Write dry-run HTML via build_dry_run_html(). + 4) If ``dry_run_only`` is False, call ``apply_indexer_moves()`` to move files. + Playlist generation can be skipped by passing ``create_playlists=False``. + Returns summary: {"moved": , "html": , "dry_run": }. + """ + if log_callback is None: + def log_callback(msg): pass + + # Ensure Not Sorted directory exists for user exclusions + not_sorted_dir = os.path.join(root_path, "Not Sorted") + os.makedirs(not_sorted_dir, exist_ok=True) + + # Phase 1–4: Generate moves, tags, and collect decision log + moves, tag_index, decision_log = compute_moves_and_tag_index( + root_path, + log_callback, + progress_callback, + dry_run=dry_run_only, + enable_phase_c=enable_phase_c, + flush_cache=flush_cache, + max_workers=max_workers, + coord=None, + ) + + # Write the detailed log file + docs_dir = os.path.join(root_path, "Docs") + os.makedirs(docs_dir, exist_ok=True) + log_path = os.path.join(docs_dir, "indexer_log.txt") + with open(log_path, "w", encoding="utf-8") as lf: + lf.write("Indexing Decision Log\n") + lf.write("======================\n") + lf.write(f"Library root: {root_path}\n") + lf.write(f"Generated on: {__import__('datetime').datetime.now()}\n\n") + for line in decision_log: + lf.write(line + "\n") + log_callback(f"✓ Detailed log written to: {log_path}") + + # Build dry-run HTML + log_callback("5/6: Writing dry-run HTML…") + build_dry_run_html( + root_path, + output_html_path, + log_callback, + enable_phase_c=enable_phase_c, + flush_cache=flush_cache, + max_workers=max_workers, + ) + + if not dry_run_only: + # Phase 5–6: Actually move audio files and cover images + actual_summary = apply_indexer_moves( + root_path, + log_callback, + progress_callback, + create_playlists=create_playlists, + ) + summary = {"moved": actual_summary["moved"], "html": output_html_path, "dry_run": False} + return summary + else: + return {"moved": 0, "html": output_html_path, "dry_run": True} + + +# ─── E. DUPLICATE DETECTION HELPER ───────────────────────────────────── + +def find_duplicates(root_path, log_callback=None): + """Return list of (original_path, duplicate_path) that would be marked as + duplicates by the indexer.""" + + moves, _, _ = compute_moves_and_tag_index( + root_path, + log_callback, + None, + dry_run=True, + enable_phase_c=False, + flush_cache=False, + max_workers=None, + coord=None, + ) + dup_indicator = os.path.join("Duplicates", "") + return [ + (old, new) + for old, new in moves.items() + if dup_indicator in new + ] + + +def main(argv: List[str] | None = None) -> None: + import argparse + + parser = argparse.ArgumentParser(description="SoundVault Music Indexer") + parser.add_argument("root", help="Library root path") + parser.add_argument("--dry-run", action="store_true", help="Preview only") + parser.add_argument("--enable-phase-c", action="store_true", help="Enable cross-album scan") + parser.add_argument("--flush-cache", action="store_true", help="Flush fingerprint cache") + parser.add_argument("--max-workers", type=int, default=None, help="Fingerprint worker count") + parser.add_argument("--no-playlists", action="store_true", help="Skip playlist creation") + args = parser.parse_args(argv) + + output = os.path.join(args.root, "Docs", "MusicIndex.html") + if args.dry_run: + build_dry_run_html( + args.root, + output, + print, + enable_phase_c=args.enable_phase_c, + flush_cache=args.flush_cache, + max_workers=args.max_workers, + ) + else: + run_full_indexer( + args.root, + output, + dry_run_only=False, + log_callback=print, + progress_callback=None, + enable_phase_c=args.enable_phase_c, + flush_cache=args.flush_cache, + max_workers=args.max_workers, + create_playlists=not args.no_playlists, + ) + + +if __name__ == "__main__": + from crash_logger import install as install_crash_logger + + install_crash_logger() + main() diff --git a/library_sync_indexer_engine/indexer_engine/near_duplicate_detector.py b/library_sync_indexer_engine/indexer_engine/near_duplicate_detector.py new file mode 100644 index 0000000..214b3e2 --- /dev/null +++ b/library_sync_indexer_engine/indexer_engine/near_duplicate_detector.py @@ -0,0 +1,256 @@ +# near_duplicate_detector.py +"""Helpers for fuzzy fingerprint grouping.""" +from __future__ import annotations + +import os +from collections import defaultdict +from typing import Dict, Set, List, Iterable, Tuple +from concurrent.futures import ThreadPoolExecutor, as_completed +import logging +import base64 + +logger = logging.getLogger(__name__) + +from music_indexer_api import _keep_score + +EXCLUSION_KEYWORDS = ['remix', 'remastered', 'edit', 'version'] + + +def _parse_fp(fp: str) -> tuple[str, List[int] | bytes] | None: + """Parse fingerprint string as integer list or base64 bytes.""" + text = fp.strip() + if not text: + return None + try: + arr = [int(x) for x in text.replace(',', ' ').split()] + if arr: + return ("ints", arr) + except Exception: + pass + try: + data = base64.urlsafe_b64decode(text + '=' * (-len(text) % 4)) + if data: + return ("bytes", data) + except Exception: + pass + return None + + +def _hamming_ints(a: List[int], b: List[int]) -> float: + n = min(len(a), len(b)) + if n == 0: + return 1.0 + diff = sum(x != y for x, y in zip(a[:n], b[:n])) + return diff / n + + +def _hamming_bytes(a: bytes, b: bytes) -> float: + n = min(len(a), len(b)) + if n == 0: + return 1.0 + diff = 0 + for i in range(n): + diff += (a[i] ^ b[i]).bit_count() + return diff / (n * 8) + + +def fingerprint_distance(fp1: str | None, fp2: str | None) -> float: + """Return normalized Hamming distance between two fingerprint strings.""" + if not fp1 or not fp2: + return 1.0 + if fp1 == fp2: + return 0.0 + p1 = _parse_fp(fp1) + p2 = _parse_fp(fp2) + if not p1 or not p2 or p1[0] != p2[0]: + return 1.0 + kind, a = p1 + _, b = p2 + if kind == "ints": + return _hamming_ints(a, b) # type: ignore[arg-type] + else: + return _hamming_bytes(a, b) # type: ignore[arg-type] + + +def _has_exclusion(info: Dict[str, str | None]) -> bool: + text = f"{info.get('title','')} {info.get('album','')}".lower() + return any(k in text for k in EXCLUSION_KEYWORDS) + + +def _scan_paths( + paths: Iterable[str], + file_infos: Dict[str, Dict[str, str | None]], + threshold: float, + log_callback, +) -> Tuple[List[Set[str]], int]: + """Scan a list of paths for near duplicate clusters.""" + adj: Dict[str, Set[str]] = defaultdict(set) + comparisons = 0 + path_list = list(paths) + for i, p in enumerate(path_list): + for q in path_list[i + 1 :]: + dist = fingerprint_distance(file_infos[p]["fp"], file_infos[q]["fp"]) + if dist <= threshold: + adj[p].add(q) + adj[q].add(p) + log_callback( + f" → near-dup {os.path.basename(p)} vs {os.path.basename(q)} dist={dist:.3f}" + ) + comparisons += 1 + + visited: Set[str] = set() + clusters: List[Set[str]] = [] + for p in path_list: + if p in visited: + continue + stack = [p] + comp = {p} + visited.add(p) + while stack: + cur = stack.pop() + for nb in adj.get(cur, []): + if nb not in visited: + visited.add(nb) + comp.add(nb) + stack.append(nb) + if len(comp) > 1: + clusters.append(comp) + return clusters, comparisons + + + + +def _scan_album( + album: str, + paths: Iterable[str], + file_infos: Dict[str, Dict[str, str | None]], + threshold: float, + log_callback, +) -> tuple[str, List[Set[str]], int]: + """Worker helper: scan a single album for near duplicates.""" + clusters, comparisons = _scan_paths(paths, file_infos, threshold, log_callback) + return album, clusters, comparisons + + +def find_near_duplicates( + file_infos: Dict[str, Dict[str, str | None]], + ext_priority: Dict[str, int], + threshold: float, + log_callback=None, + enable_cross_album: bool = False, + coord=None, + max_workers: int | None = None, +) -> Dict[str, str]: + """Return mapping of files to delete as near duplicates.""" + if log_callback is None: + def log_callback(msg: str) -> None: + pass + + paths = [p for p, info in file_infos.items() if info.get("fp") and not _has_exclusion(info)] + + by_album: Dict[str, List[str]] = defaultdict(list) + for p in paths: + album = file_infos[p].get("album") or "" + by_album[album].append(p) + + album_items = sorted(by_album.items(), key=lambda x: x[0]) + total_albums = len(album_items) + total_comparisons = 0 + clusters: List[Set[str]] = [] + html_lines = ["

      Phase B – Album Near-Duplicates

      ", "
      "]
      +
      +    with ThreadPoolExecutor(max_workers=max_workers) as ex:
      +        futures = {}
      +        for idx, (album, album_paths) in enumerate(album_items, start=1):
      +            if len(album_paths) < 2:
      +                continue
      +            logger.debug(
      +                f"Phase B: [{idx}/{total_albums}] Scanning album '{album}' ({len(album_paths)} tracks)"
      +            )
      +            html_lines.append(f"Album: {album or ''} ({len(album_paths)} tracks)")
      +            futures[ex.submit(_scan_album, album, album_paths, file_infos, threshold, log_callback)] = album
      +
      +        for fut in as_completed(futures):
      +            album = futures[fut]
      +            try:
      +                _alb, c, comps = fut.result()
      +                clusters.extend(c)
      +                total_comparisons += comps
      +                if coord is not None and c:
      +                    coord.add_near_dupe_clusters([list(s) for s in c])
      +            except Exception as e:
      +                logger.error(f"Phase B album '{album}' failed: {e}")
      +                html_lines.append(f"Error scanning album '{album}'")
      +
      +    html_lines.append("
      ") + html_str = "\n".join(html_lines) + if coord is not None: + coord.set_html_section("B", html_str) + + cross_clusters: List[Set[str]] = [] + if enable_cross_album: + cross_lines = ["

      Phase C – Cross-Album Near-Duplicates

      ", "
      "]
      +        by_song: Dict[Tuple[str, str], List[str]] = defaultdict(list)
      +        for p in paths:
      +            key = (
      +                file_infos[p].get("primary") or "",
      +                file_infos[p].get("title") or "",
      +            )
      +            by_song[key].append(p)
      +
      +        song_items = sorted(by_song.items(), key=lambda x: (x[0][0], x[0][1]))
      +        total_songs = len(song_items)
      +        with ThreadPoolExecutor(max_workers=max_workers) as ex:
      +            futures = {}
      +            for idx, ((prim, title), song_paths) in enumerate(song_items, start=1):
      +                if len(song_paths) < 2:
      +                    continue
      +                logger.debug(
      +                    f"Phase C: [{idx}/{total_songs}] Scanning '{title}' ({len(song_paths)} tracks)"
      +                )
      +                cross_lines.append(f"{title or ''} ({len(song_paths)} tracks)")
      +                futures[ex.submit(_scan_paths, song_paths, file_infos, threshold, log_callback)] = (prim, title)
      +
      +            for fut in as_completed(futures):
      +                _key = futures[fut]
      +                try:
      +                    c, comps = fut.result()
      +                    cross_clusters.extend(c)
      +                    if coord is not None and c:
      +                        coord.add_near_dupe_clusters([list(s) for s in c])
      +                    total_comparisons += comps
      +                except Exception as e:
      +                    logger.error(f"Phase C group '{_key}' failed: {e}")
      +                    cross_lines.append(f"Error scanning group '{_key}'")
      +
      +        cross_lines.append("
      ") + cross_html = "\n".join(cross_lines) + if coord is not None: + coord.set_html_section("C", cross_html) + + if logger.isEnabledFor(logging.DEBUG): + logger.debug( + f"Phase B summary: {total_albums} albums scanned, {total_comparisons} comparisons, {len(clusters)} clusters" + ) + + all_clusters = clusters + cross_clusters + + to_delete: Dict[str, str] = {} + for cluster in all_clusters: + by_alb: Dict[str, List[str]] = defaultdict(list) + for path in cluster: + album = file_infos[path].get("album") or "" + by_alb[album].append(path) + for album_paths in by_alb.values(): + if len(album_paths) <= 1: + continue + scored = sorted( + album_paths, + key=lambda p: _keep_score(p, file_infos[p], ext_priority), + reverse=True, + ) + keep = scored[0] + for loser in scored[1:]: + if loser not in to_delete: + to_delete[loser] = f"Near-duplicate of {os.path.basename(keep)}" + return to_delete diff --git a/library_sync_indexer_engine/indexer_engine/playlist_generator.py b/library_sync_indexer_engine/indexer_engine/playlist_generator.py new file mode 100644 index 0000000..2f5d669 --- /dev/null +++ b/library_sync_indexer_engine/indexer_engine/playlist_generator.py @@ -0,0 +1,188 @@ +"""Playlist generation utilities with edge-case handling. + +DEFAULT_EXTS defines which audio file extensions are included when building +playlists. Existing playlists are overwritten by default; pass +``overwrite=False`` to skip files that already exist. +""" + +import os, hashlib +from collections import defaultdict +from crash_watcher import record_event +from crash_logger import watcher + +# Default extensions for playlist generation +DEFAULT_EXTS = {".mp3", ".flac", ".wav", ".aac", ".m4a"} + + +def _sanitize_name(rel_path, existing): + """Sanitize a relative path for playlist filename. + + Replaces path separators with underscores and appends a short hash if + the sanitized name already exists in ``existing``. + """ + base = rel_path.replace(os.sep, "_") or "root" + if base not in existing: + return base + h = hashlib.md5(rel_path.encode("utf-8")).hexdigest()[:6] + return f"{base}_{h}" + + +@watcher.traced +def generate_playlists( + moves, + root_path, + output_dir=None, + valid_exts=None, + overwrite=True, + log_callback=None, +): + """Generate M3U playlists based on move mapping. + + Parameters + ---------- + moves : dict + Mapping of original file paths to their new locations. + root_path : str + Root of the music library. + output_dir : str or None + Directory to write playlists into. Defaults to ``root_path/Playlists``. + valid_exts : set or None + Extensions to include. Defaults to ``DEFAULT_EXTS``. + overwrite : bool + Whether to overwrite existing playlists. + log_callback : callable + Function for logging messages. + """ + if log_callback is None: + log_callback = lambda msg: None + valid_exts = {e.lower() for e in (valid_exts or DEFAULT_EXTS)} + + playlists_dir = output_dir or os.path.join(root_path, "Playlists") + os.makedirs(playlists_dir, exist_ok=True) + record_event(f"playlist_generator: generating playlists in {playlists_dir}") + + dir_map = defaultdict(list) + for old, new in moves.items(): + ext = os.path.splitext(new)[1].lower() + if ext in valid_exts: + dir_map[os.path.dirname(old)].append(new) + + used = set() + for old_dir, files in dir_map.items(): + if not files: + log_callback(f"\u26A0 Skipping empty folder: {old_dir}") + continue + + rel = os.path.relpath(old_dir, root_path) + name = _sanitize_name(rel, used) + used.add(name) + + playlist_file = os.path.join(playlists_dir, f"{name}.m3u") + + rel_files = {os.path.relpath(p, playlists_dir) for p in files} + + if os.path.exists(playlist_file): + if not overwrite: + try: + with open(playlist_file, "r", encoding="utf-8") as f: + existing = {line.strip() for line in f if line.strip()} + except Exception as e: + log_callback(f"\u2717 Failed to read {playlist_file}: {e}") + existing = set() + rel_files.update(existing) + log_callback(f"\u2192 Writing playlist: {playlist_file}") + else: + log_callback(f"\u2192 Writing playlist: {playlist_file}") + + try: + with open(playlist_file, "w", encoding="utf-8") as f: + for p in sorted(rel_files): + f.write(p + "\n") + except Exception as e: + log_callback(f"\u2717 Failed to write {playlist_file}: {e}") + record_event("playlist_generator: playlist generation complete") + + +def write_playlist(tracks, outfile): + """Write a simple M3U playlist from ``tracks`` to ``outfile``.""" + os.makedirs(os.path.dirname(outfile), exist_ok=True) + try: + with open(outfile, "w", encoding="utf-8") as f: + for p in tracks: + f.write(os.path.relpath(p, os.path.dirname(outfile)) + "\n") + except Exception as e: + raise RuntimeError(f"Failed to write playlist {outfile}: {e}") + + +@watcher.traced +def update_playlists(changes): + """Update ``.m3u`` playlists based on moved or deleted tracks. + + ``changes`` may be either an iterable of new track paths or a mapping of + ``old_path -> new_path``. A ``None`` value means the old path was removed + without a replacement. + """ + + if not changes: + return + + if isinstance(changes, dict): + move_map = dict(changes) + else: + move_map = {p: p for p in changes} + + all_paths = list(move_map.keys()) + [p for p in move_map.values() if p] + root = os.path.commonpath(all_paths) if all_paths else None + if not root: + return + + playlists_dir = os.path.join(root, "Playlists") + if not os.path.isdir(playlists_dir): + return + record_event("playlist_generator: updating existing playlists") + + for dirpath, _dirs, files in os.walk(playlists_dir): + for fname in files: + if not fname.lower().endswith(".m3u"): + continue + pl_path = os.path.join(dirpath, fname) + try: + with open(pl_path, "r", encoding="utf-8") as f: + lines = [ln.rstrip("\n") for ln in f] + except Exception: + continue + + changed = False + new_lines = [] + for ln in lines: + abs_line = os.path.normpath(os.path.join(dirpath, ln)) + if abs_line in move_map: + new = move_map[abs_line] + if new is None: + changed = True + continue + rel_new = os.path.relpath(new, dirpath) + if rel_new != ln: + changed = True + new_lines.append(rel_new) + else: + new_lines.append(ln) + + if changed: + try: + with open(pl_path, "w", encoding="utf-8") as f: + for l in new_lines: + f.write(l + "\n") + except Exception: + pass + + new_tracks = [p for p in move_map.values() if p] + if new_tracks: + generate_playlists( + {p: p for p in new_tracks}, + root, + overwrite=False, + log_callback=lambda m: None, + ) + record_event("playlist_generator: playlist update complete") + diff --git a/library_sync_indexer_engine/indexer_engine/utils/path_helpers.py b/library_sync_indexer_engine/indexer_engine/utils/path_helpers.py new file mode 100644 index 0000000..2aef7fc --- /dev/null +++ b/library_sync_indexer_engine/indexer_engine/utils/path_helpers.py @@ -0,0 +1,20 @@ +import os + + +def ensure_long_path(path: str) -> str: + if os.name == "nt": + path = os.path.abspath(path) + if not path.startswith("\\\\?\\"): + path = "\\\\?\\" + os.path.normpath(path) + return path + + +def strip_ext_prefix(path: str) -> str: + """Return ``path`` without a Windows extended-length prefix.""" + if os.name == "nt" and path.startswith(r"\\?\\"): + return path[4:] + return path + + +# Backwards compatibility ------------------------------------------------- +strip_long_path_prefix = strip_ext_prefix From 8593bbfac36064e015a72a99ef8488410bb5fdfd Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sat, 20 Dec 2025 13:30:44 -0500 Subject: [PATCH 135/606] Add library sync dry-run plan harness --- library_sync.py | 101 ++++++++++++++++++ .../indexer_engine/music_indexer_api.py | 71 +++++++----- music_indexer_api.py | 71 +++++++----- tests/test_library_sync_plan_preview.py | 62 +++++++++++ 4 files changed, 249 insertions(+), 56 deletions(-) create mode 100644 tests/test_library_sync_plan_preview.py diff --git a/library_sync.py b/library_sync.py index 2686fec..f66ee78 100644 --- a/library_sync.py +++ b/library_sync.py @@ -655,3 +655,104 @@ def copy_new_tracks(*_args, **_kwargs): def replace_tracks(*_args, **_kwargs): """Deprecated: blocked per review-first redesign.""" raise RuntimeError("File operations are disabled in the Library Sync review tool.") + + +@dataclass +class LibrarySyncPlan: + """Move/route plan computed by the indexer engine for Library Sync.""" + + library_root: str + incoming_root: str + destination_root: str + moves: Dict[str, str] + tag_index: Dict[str, object] + decision_log: List[str] + + def planned_moves(self) -> Dict[str, str]: + """Return a copy of the planned move mapping for execution.""" + return dict(self.moves) + + def render_preview(self, output_html_path: str) -> str: + """Generate a dry-run HTML preview from this plan.""" + heading = "Library Sync (Dry Run Preview)" + idx.render_dry_run_html_from_plan( + self.destination_root, + output_html_path, + self.moves, + self.tag_index, + heading_text=heading, + title_prefix=heading, + ) + return output_html_path + + +def compute_library_sync_plan( + library_root: str, + incoming_folder: str, + *, + log_callback: Callable[[str], None] | None = None, + progress_callback: Callable[[int, int, str, str], None] | None = None, + flush_cache: bool = False, + max_workers: int | None = None, +) -> LibrarySyncPlan: + """Compute a deterministic move/route plan for Library Sync.""" + if log_callback is None: + def log_callback(msg: str) -> None: + _dlog(msg) + if progress_callback is None: + def progress_callback(_c: int, _t: int, _m: str, _p: str) -> None: + pass + + moves, tag_index, decision_log = idx.compute_moves_and_tag_index( + incoming_folder, + log_callback, + progress_callback, + dry_run=True, + enable_phase_c=False, + flush_cache=flush_cache, + max_workers=max_workers, + coord=None, + ) + + destination_root = os.path.join(library_root, "Music") + if not os.path.isdir(destination_root): + destination_root = library_root + + def _remap_path(path: str) -> str: + rel = os.path.relpath(path, incoming_folder) + return _normalize_path(os.path.join(destination_root, rel)) + + remapped_moves = {_normalize_path(src): _remap_path(dst) for src, dst in moves.items()} + remapped_tag_index = {_remap_path(dst): data for dst, data in tag_index.items()} + + return LibrarySyncPlan( + library_root=_normalize_path(library_root), + incoming_root=_normalize_path(incoming_folder), + destination_root=_normalize_path(destination_root), + moves=remapped_moves, + tag_index=remapped_tag_index, + decision_log=decision_log, + ) + + +def build_library_sync_preview( + library_root: str, + incoming_folder: str, + output_html_path: str, + *, + log_callback: Callable[[str], None] | None = None, + progress_callback: Callable[[int, int, str, str], None] | None = None, + flush_cache: bool = False, + max_workers: int | None = None, +) -> LibrarySyncPlan: + """Compute a Library Sync plan and write the dry-run preview.""" + plan = compute_library_sync_plan( + library_root, + incoming_folder, + log_callback=log_callback, + progress_callback=progress_callback, + flush_cache=flush_cache, + max_workers=max_workers, + ) + plan.render_preview(output_html_path) + return plan diff --git a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py index 5d962bb..ecb42fd 100644 --- a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py +++ b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py @@ -757,38 +757,20 @@ def _compute(path: str) -> tuple[int | None, str | None]: # ─── B. BUILD DRY-RUN HTML ───────────────────────────────────────────── -def build_dry_run_html( +def render_dry_run_html_from_plan( root_path, output_html_path, - log_callback=None, - enable_phase_c=False, - flush_cache=False, - max_workers=None, -): - """ - 1) Call compute_moves_and_tag_index() to get (moves, tag_index, decision_log). - 2) Write a dry-run HTML to output_html_path showing the proposed tree. - Does NOT move any files. - """ - if log_callback is None: - def log_callback(msg): pass - + moves, + tag_index, + *, + heading_text="Phase A – Exact Metadata", + title_prefix="Music Index (Dry Run)", +): + """Render a dry-run preview from an already computed move plan.""" coord = DryRunCoordinator() - moves, tag_index, _ = compute_moves_and_tag_index( - root_path, - log_callback, - None, - dry_run=True, - enable_phase_c=enable_phase_c, - flush_cache=flush_cache, - max_workers=max_workers, - coord=coord, - ) - - log_callback("5/6: Writing dry-run HTML…") def build_exact_metadata_section() -> str: - lines = ["

      Phase A – Exact Metadata

      ", "
      "]
      +        lines = [f"

      {heading_text}

      ", "
      "]
               lines.append(f"{sanitize(os.path.basename(root_path))}/")
               tree_nodes = set()
               for new_path in moves.values():
      @@ -822,14 +804,47 @@ def build_exact_metadata_section() -> str:
           with open(output_html_path, "w", encoding="utf-8") as out:
               out.write(
                   "\n\n\n  \n  "
      -            f"Music Index (Dry Run) – {sanitize(os.path.basename(root_path))}\n  "
      +            f"{title_prefix} – {sanitize(os.path.basename(root_path))}\n  "
                   "\n\n\n"
               )
               out.write(html_body)
               if html_body and not html_body.endswith("\n"):
                   out.write("\n")
               out.write("\n\n")
      +    return output_html_path
      +
      +
      +def build_dry_run_html(
      +    root_path,
      +    output_html_path,
      +    log_callback=None,
      +    enable_phase_c=False,
      +    flush_cache=False,
      +    max_workers=None,
      +): 
      +    """
      +    1) Call compute_moves_and_tag_index() to get (moves, tag_index, decision_log).
      +    2) Write a dry-run HTML to output_html_path showing the proposed tree.
      +    Does NOT move any files.
      +    """
      +    if log_callback is None:
      +        def log_callback(msg): pass
      +
      +    coord = DryRunCoordinator()
      +    moves, tag_index, _ = compute_moves_and_tag_index(
      +        root_path,
      +        log_callback,
      +        None,
      +        dry_run=True,
      +        enable_phase_c=enable_phase_c,
      +        flush_cache=flush_cache,
      +        max_workers=max_workers,
      +        coord=coord,
      +    )
      +
      +    log_callback("5/6: Writing dry-run HTML…")
       
      +    render_dry_run_html_from_plan(root_path, output_html_path, moves, tag_index)
           log_callback(f"✓ Dry-run HTML written to: {output_html_path}")
           return {"moved": 0, "html": output_html_path, "dry_run": True}
       
      diff --git a/music_indexer_api.py b/music_indexer_api.py
      index 5d962bb..ecb42fd 100644
      --- a/music_indexer_api.py
      +++ b/music_indexer_api.py
      @@ -757,38 +757,20 @@ def _compute(path: str) -> tuple[int | None, str | None]:
       
       # ─── B. BUILD DRY-RUN HTML ─────────────────────────────────────────────
       
      -def build_dry_run_html(
      +def render_dry_run_html_from_plan(
           root_path,
           output_html_path,
      -    log_callback=None,
      -    enable_phase_c=False,
      -    flush_cache=False,
      -    max_workers=None,
      -): 
      -    """
      -    1) Call compute_moves_and_tag_index() to get (moves, tag_index, decision_log).
      -    2) Write a dry-run HTML to output_html_path showing the proposed tree.
      -    Does NOT move any files.
      -    """
      -    if log_callback is None:
      -        def log_callback(msg): pass
      -
      +    moves,
      +    tag_index,
      +    *,
      +    heading_text="Phase A – Exact Metadata",
      +    title_prefix="Music Index (Dry Run)",
      +):
      +    """Render a dry-run preview from an already computed move plan."""
           coord = DryRunCoordinator()
      -    moves, tag_index, _ = compute_moves_and_tag_index(
      -        root_path,
      -        log_callback,
      -        None,
      -        dry_run=True,
      -        enable_phase_c=enable_phase_c,
      -        flush_cache=flush_cache,
      -        max_workers=max_workers,
      -        coord=coord,
      -    )
      -
      -    log_callback("5/6: Writing dry-run HTML…")
       
           def build_exact_metadata_section() -> str:
      -        lines = ["

      Phase A – Exact Metadata

      ", "
      "]
      +        lines = [f"

      {heading_text}

      ", "
      "]
               lines.append(f"{sanitize(os.path.basename(root_path))}/")
               tree_nodes = set()
               for new_path in moves.values():
      @@ -822,14 +804,47 @@ def build_exact_metadata_section() -> str:
           with open(output_html_path, "w", encoding="utf-8") as out:
               out.write(
                   "\n\n\n  \n  "
      -            f"Music Index (Dry Run) – {sanitize(os.path.basename(root_path))}\n  "
      +            f"{title_prefix} – {sanitize(os.path.basename(root_path))}\n  "
                   "\n\n\n"
               )
               out.write(html_body)
               if html_body and not html_body.endswith("\n"):
                   out.write("\n")
               out.write("\n\n")
      +    return output_html_path
      +
      +
      +def build_dry_run_html(
      +    root_path,
      +    output_html_path,
      +    log_callback=None,
      +    enable_phase_c=False,
      +    flush_cache=False,
      +    max_workers=None,
      +): 
      +    """
      +    1) Call compute_moves_and_tag_index() to get (moves, tag_index, decision_log).
      +    2) Write a dry-run HTML to output_html_path showing the proposed tree.
      +    Does NOT move any files.
      +    """
      +    if log_callback is None:
      +        def log_callback(msg): pass
      +
      +    coord = DryRunCoordinator()
      +    moves, tag_index, _ = compute_moves_and_tag_index(
      +        root_path,
      +        log_callback,
      +        None,
      +        dry_run=True,
      +        enable_phase_c=enable_phase_c,
      +        flush_cache=flush_cache,
      +        max_workers=max_workers,
      +        coord=coord,
      +    )
      +
      +    log_callback("5/6: Writing dry-run HTML…")
       
      +    render_dry_run_html_from_plan(root_path, output_html_path, moves, tag_index)
           log_callback(f"✓ Dry-run HTML written to: {output_html_path}")
           return {"moved": 0, "html": output_html_path, "dry_run": True}
       
      diff --git a/tests/test_library_sync_plan_preview.py b/tests/test_library_sync_plan_preview.py
      new file mode 100644
      index 0000000..41bcb57
      --- /dev/null
      +++ b/tests/test_library_sync_plan_preview.py
      @@ -0,0 +1,62 @@
      +import os
      +import sys
      +import types
      +
      +# Provide simple stubs for mutagen so the module imports
      +mutagen_stub = types.ModuleType("mutagen")
      +
      +
      +class DummyAudio:
      +    def __init__(self):
      +        self.tags = None
      +        self.pictures = []
      +
      +
      +def File(*_a, **_k):
      +    return DummyAudio()
      +
      +
      +mutagen_stub.File = File
      +
      +id3_stub = types.ModuleType("id3")
      +
      +
      +class DummyID3Error(Exception):
      +    pass
      +
      +
      +id3_stub.ID3NoHeaderError = DummyID3Error
      +mutagen_stub.id3 = id3_stub
      +sys.modules.setdefault("mutagen", mutagen_stub)
      +sys.modules.setdefault("mutagen.id3", id3_stub)
      +acoustid_stub = types.SimpleNamespace(fingerprint_file=lambda *_a, **_k: (None, None))
      +sys.modules.setdefault("acoustid", acoustid_stub)
      +
      +from library_sync import compute_library_sync_plan
      +
      +
      +def test_library_sync_plan_preview_reuses_plan(tmp_path):
      +    library_root = tmp_path / "library"
      +    incoming_root = tmp_path / "incoming"
      +    library_root.mkdir()
      +    incoming_root.mkdir()
      +    music_root = library_root / "Music"
      +    music_root.mkdir()
      +
      +    incoming_song = incoming_root / "artist - track.mp3"
      +    incoming_song.write_text("audio")
      +
      +    plan = compute_library_sync_plan(str(library_root), str(incoming_root))
      +
      +    assert plan.destination_root == os.path.normpath(str(music_root))
      +    assert str(incoming_song) in plan.moves
      +    dest_path = plan.moves[str(incoming_song)]
      +    assert dest_path.startswith(str(music_root))
      +
      +    html_path = tmp_path / "preview.html"
      +    plan.render_preview(str(html_path))
      +    content = html_path.read_text()
      +
      +    assert "Library Sync (Dry Run Preview)" in content
      +    assert incoming_song.name in content
      +    assert f"{music_root.name}/" in content
      
      From a09a12590994a9c8de62b91feb12b0082ae8fa7b Mon Sep 17 00:00:00 2001
      From: Greensand321 
      Date: Sat, 20 Dec 2025 13:45:36 -0500
      Subject: [PATCH 136/606] Add library sync plan preview and execution controls
      
      ---
       library_sync.py                         |  77 +++++-
       library_sync_review.py                  | 301 +++++++++++++++++++++++-
       tests/test_library_sync_plan_preview.py |  44 +++-
       3 files changed, 418 insertions(+), 4 deletions(-)
      
      diff --git a/library_sync.py b/library_sync.py
      index f66ee78..1464a52 100644
      --- a/library_sync.py
      +++ b/library_sync.py
      @@ -5,6 +5,7 @@
       import json
       import logging
       import os
      +import shutil
       import threading
       import time
       from dataclasses import dataclass, field
      @@ -27,6 +28,7 @@
       
       from crash_watcher import record_event
       from crash_logger import watcher
      +from indexer_control import IndexCancelled
       
       debug: bool = False
       _logger = logging.getLogger(__name__)
      @@ -694,8 +696,10 @@ def compute_library_sync_plan(
           progress_callback: Callable[[int, int, str, str], None] | None = None,
           flush_cache: bool = False,
           max_workers: int | None = None,
      +    cancel_event: threading.Event | None = None,
       ) -> LibrarySyncPlan:
           """Compute a deterministic move/route plan for Library Sync."""
      +    cancel_event = cancel_event or threading.Event()
           if log_callback is None:
               def log_callback(msg: str) -> None:
                   _dlog(msg)
      @@ -703,10 +707,23 @@ def log_callback(msg: str) -> None:
               def progress_callback(_c: int, _t: int, _m: str, _p: str) -> None:
                   pass
       
      +    def _cancellable_progress(current: int, total: int, message: str, phase: str) -> None:
      +        if cancel_event.is_set():
      +            raise IndexCancelled()
      +        progress_callback(current, total, message, phase)
      +
      +    def _cancellable_log(message: str) -> None:
      +        if cancel_event.is_set():
      +            raise IndexCancelled()
      +        log_callback(message)
      +
      +    if cancel_event.is_set():
      +        raise IndexCancelled()
      +
           moves, tag_index, decision_log = idx.compute_moves_and_tag_index(
               incoming_folder,
      -        log_callback,
      -        progress_callback,
      +        _cancellable_log,
      +        _cancellable_progress,
               dry_run=True,
               enable_phase_c=False,
               flush_cache=flush_cache,
      @@ -744,6 +761,7 @@ def build_library_sync_preview(
           progress_callback: Callable[[int, int, str, str], None] | None = None,
           flush_cache: bool = False,
           max_workers: int | None = None,
      +    cancel_event: threading.Event | None = None,
       ) -> LibrarySyncPlan:
           """Compute a Library Sync plan and write the dry-run preview."""
           plan = compute_library_sync_plan(
      @@ -753,6 +771,61 @@ def build_library_sync_preview(
               progress_callback=progress_callback,
               flush_cache=flush_cache,
               max_workers=max_workers,
      +        cancel_event=cancel_event,
           )
           plan.render_preview(output_html_path)
           return plan
      +
      +
      +def execute_library_sync_plan(
      +    plan: LibrarySyncPlan,
      +    *,
      +    log_callback: Callable[[str], None] | None = None,
      +    progress_callback: Callable[[int, int, str, str], None] | None = None,
      +    cancel_event: threading.Event | None = None,
      +) -> Dict[str, object]:
      +    """Apply a previously computed Library Sync plan.
      +
      +    The execution routine only trusts the provided plan. No recomputation or
      +    reinterpretation of moves occurs here, ensuring the same routing used for
      +    preview is applied during execution.
      +    """
      +
      +    cancel_event = cancel_event or threading.Event()
      +
      +    def _log(message: str) -> None:
      +        if log_callback:
      +            log_callback(message)
      +
      +    def _progress(current: int, total: int, path: str, phase: str) -> None:
      +        if progress_callback:
      +            progress_callback(current, total, path, phase)
      +
      +    if cancel_event.is_set():
      +        _log("✘ Execution cancelled before start.")
      +        return {"moved": 0, "errors": [], "cancelled": True}
      +
      +    moves = sorted(plan.moves.items())
      +    total = len(moves)
      +    summary: Dict[str, object] = {"moved": 0, "errors": [], "cancelled": False}
      +    _log(f"Executing plan with {total} moves…")
      +
      +    for idx, (src, dst) in enumerate(moves, start=1):
      +        if cancel_event.is_set():
      +            summary["cancelled"] = True
      +            _log("✘ Execution cancelled.")
      +            break
      +
      +        _progress(idx, total, src, "execute")
      +        try:
      +            os.makedirs(os.path.dirname(dst), exist_ok=True)
      +            shutil.move(src, dst)
      +            summary["moved"] = int(summary["moved"]) + 1
      +        except Exception as exc:  # pragma: no cover - OS interaction
      +            msg = f"Failed to move {src} → {dst}: {exc}"
      +            summary.setdefault("errors", []).append(msg)
      +            _log(msg)
      +
      +    if not summary.get("cancelled"):
      +        _log("✓ Execution complete.")
      +    return summary
      diff --git a/library_sync_review.py b/library_sync_review.py
      index 6bcb38d..6f41b18 100644
      --- a/library_sync_review.py
      +++ b/library_sync_review.py
      @@ -4,16 +4,18 @@
       import json
       import os
       import threading
      +import webbrowser
       from dataclasses import dataclass, field
       from datetime import datetime
       from enum import Enum
      -from typing import Any, Callable, Dict, Iterable, List, Sequence
      +from typing import Any, Callable, Dict, Iterable, List, Sequence, Tuple
       
       import tkinter as tk
       from tkinter import filedialog, messagebox, ttk
       
       from config import DEFAULT_FP_THRESHOLDS, FORMAT_PRIORITY, load_config, save_config
       import library_sync
      +from indexer_control import IndexCancelled
       from library_sync import MatchResult, MatchStatus, TrackRecord
       from library_sync_review_report import DEFAULT_REPORT_VERSION
       from library_sync_review_state import ReviewStateStore
      @@ -205,6 +207,8 @@ def __init__(
               self.report_version_var = tk.StringVar(value=str(self.session.exported_report_version))
               self.scan_state_var = tk.StringVar(value=self._describe_state(self.session.scan_state))
               self._on_close_callback = on_close
      +        self.library_var.trace_add("write", lambda *_: self._invalidate_plan("Library folder changed"))
      +        self.incoming_var.trace_add("write", lambda *_: self._invalidate_plan("Incoming folder changed"))
       
               # Scan + progress state
               self.library_cancel = threading.Event()
      @@ -242,6 +246,18 @@ def __init__(
               self.log_entries: list[dict[str, object]] = []
               self.LOG_LIMIT = 400
       
      +        # Plan + execution state
      +        self.plan_progress_var = tk.DoubleVar(value=0)
      +        self.plan_progress_label = tk.StringVar(value="Idle")
      +        self.plan_status_var = tk.StringVar(value="No plan built.")
      +        self.plan_cancel = threading.Event()
      +        self.plan_running = False
      +        self.plan_preview_path: str | None = None
      +        self.active_plan: library_sync.LibrarySyncPlan | None = None
      +        self._plan_sources: Tuple[str, str] | None = None
      +        self._plan_version = 0
      +        self._preview_version = -1
      +
               self._build_ui()
       
           def _build_ui(self) -> None:
      @@ -317,6 +333,10 @@ def _on_canvas_config(event):
               self._build_scan_column(controls, "Existing Library", "library", 0)
               self._build_scan_column(controls, "Incoming Folder", "incoming", 1)
       
      +        plan_frame = ttk.LabelFrame(root, text="Plan & Execution")
      +        plan_frame.pack(fill="x", **padding)
      +        self._build_plan_controls(plan_frame)
      +
               body = ttk.Frame(root)
               body.pack(fill="both", expand=True, **padding)
               body.columnconfigure(0, weight=1)
      @@ -387,6 +407,28 @@ def _build_scan_column(self, parent: ttk.Frame, title: str, kind: str, column: i
                   command=lambda k=kind: self._cancel_scan(k),
               ).pack(side="left", padx=(5, 0))
       
      +    def _build_plan_controls(self, parent: ttk.Frame) -> None:
      +        ttk.Label(parent, textvariable=self.plan_status_var).pack(anchor="w", padx=5, pady=(5, 2))
      +        ttk.Progressbar(parent, variable=self.plan_progress_var, maximum=1).pack(fill="x", padx=5)
      +        ttk.Label(parent, textvariable=self.plan_progress_label, foreground="#555").pack(
      +            anchor="w", padx=5, pady=(0, 5)
      +        )
      +        btns = ttk.Frame(parent)
      +        btns.pack(fill="x", padx=5, pady=(0, 6))
      +        self.build_plan_btn = ttk.Button(btns, text="Build Plan", command=self._build_plan)
      +        self.build_plan_btn.pack(side="left")
      +        self.preview_plan_btn = ttk.Button(btns, text="Preview", command=self._preview_plan)
      +        self.preview_plan_btn.pack(side="left", padx=(5, 0))
      +        self.execute_plan_btn = ttk.Button(btns, text="Execute", command=self._execute_plan)
      +        self.execute_plan_btn.pack(side="left", padx=(5, 0))
      +        self.open_preview_btn = ttk.Button(
      +            btns, text="Open Preview", command=self._open_preview_file, state="disabled"
      +        )
      +        self.open_preview_btn.pack(side="right")
      +        self.cancel_plan_btn = ttk.Button(btns, text="Cancel", command=self._cancel_plan_task, state="disabled")
      +        self.cancel_plan_btn.pack(side="right", padx=(5, 0))
      +        self._refresh_plan_actions()
      +
           def _build_incoming_tree(self, parent: ttk.Frame) -> None:
               columns = ("name", "chips", "distance")
               tree = ttk.Treeview(parent, columns=columns, show="headings", selectmode="browse")
      @@ -486,6 +528,263 @@ def _persist_session(self) -> bool:
               )
               return True
       
      +    def _invalidate_plan(self, reason: str | None = None) -> None:
      +        """Clear any cached plan/preview when inputs change."""
      +        self.active_plan = None
      +        self._plan_sources = None
      +        self.plan_preview_path = None
      +        self._plan_version = 0
      +        self._preview_version = -1
      +        self.plan_progress_var.set(0)
      +        self.plan_progress_label.set("Idle")
      +        if reason:
      +            self.plan_status_var.set(reason)
      +        self._refresh_plan_actions()
      +
      +    def _refresh_plan_actions(self) -> None:
      +        """Enable/disable plan controls based on current state."""
      +        running = self.plan_running
      +        library_root = self.library_var.get().strip()
      +        incoming_root = self.incoming_var.get().strip()
      +        sources_match = self._plan_sources == (library_root, incoming_root)
      +        execute_ready = (
      +            not running
      +            and self.active_plan is not None
      +            and self._preview_version == self._plan_version
      +            and sources_match
      +        )
      +
      +        for btn in (self.build_plan_btn, self.preview_plan_btn):
      +            btn_state = "disabled" if running else "normal"
      +            btn.config(state=btn_state)
      +        self.cancel_plan_btn.config(state="normal" if running else "disabled")
      +        self.execute_plan_btn.config(state="normal" if execute_ready else "disabled")
      +        self.open_preview_btn.config(state="normal" if self.plan_preview_path else "disabled")
      +
      +    def _plan_inputs(self) -> Tuple[str, str] | None:
      +        """Validate folder selections for planning/execution."""
      +        library_root = self.library_var.get().strip()
      +        incoming_root = self.incoming_var.get().strip()
      +        if not library_root or not incoming_root:
      +            messagebox.showerror("Missing Folder", "Select both the Existing Library and Incoming Folder first.")
      +            return None
      +        return library_root, incoming_root
      +
      +    def _update_plan_progress(self, current: int, total: int, path: str, phase: str) -> None:
      +        if total:
      +            self.plan_progress_var.set(current / total)
      +        label = f"{phase.title()}: {os.path.basename(path) if path else ''}".strip()
      +        self.plan_progress_label.set(label or "Working…")
      +
      +    def _start_plan_task(self, render_preview: bool) -> None:
      +        inputs = self._plan_inputs()
      +        if not inputs or self.plan_running:
      +            return
      +        library_root, incoming_root = inputs
      +        self.plan_cancel.clear()
      +        self.plan_running = True
      +        self.plan_progress_var.set(0)
      +        self.plan_progress_label.set("Starting…")
      +        self.plan_status_var.set("Building preview…" if render_preview else "Building plan…")
      +        self._refresh_plan_actions()
      +        thread = threading.Thread(target=self._plan_worker, args=(library_root, incoming_root, render_preview), daemon=True)
      +        thread.start()
      +
      +    def _plan_worker(self, library_root: str, incoming_root: str, render_preview: bool) -> None:
      +        docs_dir = os.path.join(library_root, "Docs")
      +        os.makedirs(docs_dir, exist_ok=True)
      +        output_html = os.path.join(docs_dir, "LibrarySyncPreview.html")
      +
      +        def log(msg: str) -> None:
      +            self._log_event("plan_log", msg)
      +
      +        def progress(current: int, total: int, path: str, phase: str) -> None:
      +            self.after(0, lambda c=current, t=total, p=path, ph=phase: self._update_plan_progress(c, t, p, ph))
      +
      +        try:
      +            if render_preview:
      +                plan = library_sync.build_library_sync_preview(
      +                    library_root,
      +                    incoming_root,
      +                    output_html,
      +                    log_callback=log,
      +                    progress_callback=progress,
      +                    cancel_event=self.plan_cancel,
      +                )
      +                preview_path = output_html
      +            else:
      +                plan = library_sync.compute_library_sync_plan(
      +                    library_root,
      +                    incoming_root,
      +                    log_callback=log,
      +                    progress_callback=progress,
      +                    cancel_event=self.plan_cancel,
      +                )
      +                preview_path = None
      +        except IndexCancelled:
      +            self.after(0, self._on_plan_cancelled)
      +            return
      +        except Exception as exc:  # pragma: no cover - UI behavior
      +            self.after(0, lambda e=exc: self._handle_plan_error(e))
      +            return
      +
      +        self.after(
      +            0,
      +            lambda p=plan, preview=preview_path, sources=(library_root, incoming_root), render=render_preview: self._on_plan_ready(
      +                p, preview, sources, render
      +            ),
      +        )
      +
      +    def _on_plan_ready(
      +        self,
      +        plan: library_sync.LibrarySyncPlan,
      +        preview_path: str | None,
      +        sources: Tuple[str, str],
      +        render_preview: bool,
      +    ) -> None:
      +        self.plan_running = False
      +        self.plan_cancel.clear()
      +        self.active_plan = plan
      +        self._plan_sources = sources
      +        self._plan_version += 1
      +        if render_preview:
      +            self._preview_version = self._plan_version
      +            self.plan_preview_path = preview_path
      +            self.plan_status_var.set(f"Preview ready: {preview_path}")
      +            self._log_event("plan_preview", "Preview ready", {"path": preview_path, "moves": len(plan.moves)})
      +            if preview_path:
      +                try:
      +                    webbrowser.open(preview_path)
      +                except Exception:
      +                    pass
      +        else:
      +            self.plan_preview_path = None
      +            self._preview_version = -1
      +            self.plan_status_var.set("Plan built. Run Preview to inspect before execution.")
      +            self._log_event("plan_built", "Plan ready", {"moves": len(plan.moves)})
      +
      +        self.plan_progress_var.set(1)
      +        self.plan_progress_label.set("Complete")
      +        self.session.library_root, self.session.incoming_root = sources
      +        save_scan_session(self.session)
      +        self._refresh_plan_actions()
      +
      +    def _handle_plan_error(self, exc: Exception) -> None:
      +        self.plan_running = False
      +        self.plan_cancel.clear()
      +        self.plan_progress_var.set(0)
      +        self.plan_progress_label.set("Error")
      +        self.plan_status_var.set(f"Plan failed: {exc}")
      +        self._log_event("plan_error", str(exc))
      +        messagebox.showerror("Library Sync", str(exc))
      +        self._refresh_plan_actions()
      +
      +    def _on_plan_cancelled(self) -> None:
      +        self.plan_running = False
      +        self.plan_cancel.clear()
      +        self.plan_progress_var.set(0)
      +        self.plan_progress_label.set("Cancelled")
      +        self.plan_status_var.set("Plan cancelled.")
      +        self._log_event("plan_cancelled", "Plan or preview cancelled")
      +        self._refresh_plan_actions()
      +
      +    def _cancel_plan_task(self) -> None:
      +        if self.plan_running:
      +            self.plan_cancel.set()
      +            self.plan_progress_label.set("Cancelling…")
      +            self.plan_status_var.set("Cancellation requested…")
      +            self._log_event("plan_cancel_requested", "Cancellation requested")
      +        else:
      +            self.plan_cancel.clear()
      +
      +    def _build_plan(self) -> None:
      +        self._start_plan_task(render_preview=False)
      +
      +    def _preview_plan(self) -> None:
      +        self._start_plan_task(render_preview=True)
      +
      +    def _open_preview_file(self) -> None:
      +        if not self.plan_preview_path:
      +            messagebox.showinfo("Preview", "No preview has been generated yet.")
      +            return
      +        if not os.path.exists(self.plan_preview_path):
      +            messagebox.showerror("Preview Missing", "The preview file could not be found. Build a new preview.")
      +            return
      +        try:
      +            webbrowser.open(self.plan_preview_path)
      +        except Exception as exc:  # pragma: no cover - OS interaction
      +            messagebox.showerror("Preview", str(exc))
      +
      +    def _execute_plan(self) -> None:
      +        if self.plan_running:
      +            return
      +        inputs = self._plan_inputs()
      +        if not inputs:
      +            return
      +        if not self.active_plan:
      +            messagebox.showerror("Library Sync", "Build and preview a plan before executing.")
      +            return
      +        if self._plan_sources != inputs:
      +            messagebox.showerror("Library Sync", "Plan inputs changed. Rebuild and preview the plan before executing.")
      +            return
      +        if self._preview_version != self._plan_version:
      +            messagebox.showerror("Library Sync", "Generate a preview to lock the current plan before execution.")
      +            return
      +
      +        self.plan_cancel.clear()
      +        self.plan_running = True
      +        self.plan_progress_var.set(0)
      +        self.plan_progress_label.set("Executing…")
      +        self.plan_status_var.set("Executing plan…")
      +        self._refresh_plan_actions()
      +        thread = threading.Thread(target=self._execute_plan_worker, args=(self.active_plan,), daemon=True)
      +        thread.start()
      +
      +    def _execute_plan_worker(self, plan: library_sync.LibrarySyncPlan) -> None:
      +        def log(msg: str) -> None:
      +            self._log_event("execute_log", msg)
      +
      +        def progress(current: int, total: int, path: str, phase: str) -> None:
      +            self.after(0, lambda c=current, t=total, p=path, ph=phase: self._update_plan_progress(c, t, p, ph))
      +
      +        try:
      +            summary = library_sync.execute_library_sync_plan(
      +                plan,
      +                log_callback=log,
      +                progress_callback=progress,
      +                cancel_event=self.plan_cancel,
      +            )
      +        except Exception as exc:  # pragma: no cover - UI behavior
      +            self.after(0, lambda e=exc: self._handle_plan_error(e))
      +            return
      +
      +        self.after(0, lambda s=summary: self._on_execute_complete(s))
      +
      +    def _on_execute_complete(self, summary: Dict[str, object]) -> None:
      +        self.plan_running = False
      +        self.plan_cancel.clear()
      +        cancelled = bool(summary.get("cancelled"))
      +        moved = int(summary.get("moved", 0))
      +        errors = summary.get("errors", []) or []
      +
      +        if cancelled:
      +            status = "Execution cancelled."
      +            self.plan_progress_var.set(0)
      +            self.plan_progress_label.set("Cancelled")
      +            self._log_event("execute_cancelled", status)
      +        else:
      +            status = f"Execution complete. Moved {moved} files."
      +            self.plan_progress_var.set(1)
      +            self.plan_progress_label.set("Complete")
      +            self._log_event("execute_complete", status, {"moved": moved})
      +
      +        self.plan_status_var.set(status)
      +        if errors:
      +            messagebox.showerror("Execution Errors", "\n".join(errors))
      +        elif not cancelled:
      +            messagebox.showinfo("Execution Complete", status)
      +        self._refresh_plan_actions()
      +
           def _describe_state(self, state: ScanState) -> str:
               return state.value.replace("_", " ").title()
       
      diff --git a/tests/test_library_sync_plan_preview.py b/tests/test_library_sync_plan_preview.py
      index 41bcb57..1168f87 100644
      --- a/tests/test_library_sync_plan_preview.py
      +++ b/tests/test_library_sync_plan_preview.py
      @@ -1,5 +1,6 @@
       import os
       import sys
      +import threading
       import types
       
       # Provide simple stubs for mutagen so the module imports
      @@ -32,7 +33,7 @@ class DummyID3Error(Exception):
       acoustid_stub = types.SimpleNamespace(fingerprint_file=lambda *_a, **_k: (None, None))
       sys.modules.setdefault("acoustid", acoustid_stub)
       
      -from library_sync import compute_library_sync_plan
      +from library_sync import compute_library_sync_plan, execute_library_sync_plan
       
       
       def test_library_sync_plan_preview_reuses_plan(tmp_path):
      @@ -60,3 +61,44 @@ def test_library_sync_plan_preview_reuses_plan(tmp_path):
           assert "Library Sync (Dry Run Preview)" in content
           assert incoming_song.name in content
           assert f"{music_root.name}/" in content
      +
      +
      +def test_execute_library_sync_plan_moves_files(tmp_path):
      +    library_root = tmp_path / "library"
      +    incoming_root = tmp_path / "incoming"
      +    library_root.mkdir()
      +    incoming_root.mkdir()
      +    (library_root / "Music").mkdir()
      +
      +    incoming_song = incoming_root / "artist - track.mp3"
      +    incoming_song.write_text("audio")
      +
      +    plan = compute_library_sync_plan(str(library_root), str(incoming_root))
      +    dest_path = plan.moves[str(incoming_song)]
      +
      +    summary = execute_library_sync_plan(plan)
      +
      +    assert summary["moved"] == 1
      +    assert not summary.get("cancelled")
      +    assert dest_path and os.path.exists(dest_path)
      +    assert not incoming_song.exists()
      +
      +
      +def test_execute_library_sync_plan_honors_cancellation(tmp_path):
      +    library_root = tmp_path / "library"
      +    incoming_root = tmp_path / "incoming"
      +    library_root.mkdir()
      +    incoming_root.mkdir()
      +    (library_root / "Music").mkdir()
      +
      +    incoming_song = incoming_root / "artist - track.mp3"
      +    incoming_song.write_text("audio")
      +
      +    plan = compute_library_sync_plan(str(library_root), str(incoming_root))
      +    cancel_event = threading.Event()
      +    cancel_event.set()
      +
      +    summary = execute_library_sync_plan(plan, cancel_event=cancel_event)
      +
      +    assert summary["cancelled"]
      +    assert incoming_song.exists()
      
      From 8ea82903df5837b8b259b15bdf3908c02f45e3d5 Mon Sep 17 00:00:00 2001
      From: Greensand321 
      Date: Sat, 20 Dec 2025 14:00:09 -0500
      Subject: [PATCH 137/606] Add audited execution for library sync plans
      
      ---
       library_sync.py                         | 282 ++++++++++++++++++++++--
       tests/test_library_sync_execute_plan.py | 136 ++++++++++++
       2 files changed, 394 insertions(+), 24 deletions(-)
       create mode 100644 tests/test_library_sync_execute_plan.py
      
      diff --git a/library_sync.py b/library_sync.py
      index 1464a52..7a3a9d5 100644
      --- a/library_sync.py
      +++ b/library_sync.py
      @@ -9,6 +9,7 @@
       import threading
       import time
       from dataclasses import dataclass, field
      +from datetime import datetime, timezone
       from enum import Enum
       from typing import Callable, Dict, Iterable, List, Optional, Tuple
       
      @@ -19,6 +20,7 @@
           music_indexer_api as idx,
           near_duplicate_detector as idx_near_duplicate_detector,
       )
      +from playlist_generator import write_playlist
       
       fingerprint_generator = idx_fingerprint_generator
       fingerprint_distance = idx_near_duplicate_detector.fingerprint_distance
      @@ -669,6 +671,8 @@ class LibrarySyncPlan:
           moves: Dict[str, str]
           tag_index: Dict[str, object]
           decision_log: List[str]
      +    copy_only: set[str] = field(default_factory=set)
      +    allowed_replacements: set[str] = field(default_factory=set)
       
           def planned_moves(self) -> Dict[str, str]:
               """Return a copy of the planned move mapping for execution."""
      @@ -777,55 +781,285 @@ def build_library_sync_preview(
           return plan
       
       
      -def execute_library_sync_plan(
      +def _ensure_docs_dir(root: str) -> str:
      +    docs_dir = os.path.join(root, "Docs")
      +    os.makedirs(docs_dir, exist_ok=True)
      +    return docs_dir
      +
      +
      +def _render_execution_report(
      +    plan: LibrarySyncPlan,
      +    executed_moves: Dict[str, str],
      +    skipped: List[Dict[str, str]],
      +    failed: List[Dict[str, str]],
      +    review_required: List[Dict[str, str]],
      +    output_html_path: str,
      +    heading: str,
      +    dry_run: bool,
      +) -> str:
      +    """Write an executed-plan HTML using the indexer preview style."""
      +    idx.render_dry_run_html_from_plan(
      +        plan.destination_root,
      +        output_html_path,
      +        executed_moves,
      +        plan.tag_index,
      +        heading_text=heading,
      +        title_prefix=heading,
      +    )
      +
      +    sections: List[str] = []
      +    if dry_run:
      +        sections.append("

      Dry run: no files were modified.

      ") + + def _format_section(title: str, items: List[Dict[str, str]]) -> None: + if not items: + return + sections.append(f"

      {idx.sanitize(title)}

      ") + sections.append("
        ") + for item in items: + src = idx.sanitize(os.path.basename(item.get("source", ""))) + dst = idx.sanitize(os.path.basename(item.get("destination", ""))) + reason = idx.sanitize(item.get("reason", "")) + line = f"
      • {src}{dst}" + if reason: + line += f" — {reason}" + line += "
      • " + sections.append(line) + sections.append("
      ") + + _format_section("Review Required", review_required) + _format_section("Skipped", skipped) + _format_section("Failed", failed) + + if sections: + extras = "\n".join(sections) + try: + with open(output_html_path, "r+", encoding="utf-8") as f: + content = f.read() + marker = "" + if marker in content: + content = content.replace(marker, extras + "\n" + marker, 1) + else: + content += extras + f.seek(0) + f.write(content) + f.truncate() + except Exception: + pass + return output_html_path + + +def execute_plan( plan: LibrarySyncPlan, + dry_run: bool = False, *, + allow_replacements: bool | Iterable[str] | None = None, + audit_path: str | None = None, + executed_report_path: str | None = None, + playlist_path: str | None = None, + create_playlist: bool = True, log_callback: Callable[[str], None] | None = None, progress_callback: Callable[[int, int, str, str], None] | None = None, cancel_event: threading.Event | None = None, ) -> Dict[str, object]: - """Apply a previously computed Library Sync plan. + """Execute a Library Sync plan with auditing and HTML reporting. - The execution routine only trusts the provided plan. No recomputation or - reinterpretation of moves occurs here, ensuring the same routing used for - preview is applied during execution. + The execution routine performs only the operations expressed by the plan, + capturing a JSON audit trail and an executed HTML report. Existing files + are only replaced when explicitly allowed (via ``allow_replacements`` or + ``plan.allowed_replacements``). Backups are written for any destructive + action before the destination is overwritten. """ + log_callback = log_callback or (lambda _msg: None) + progress_callback = progress_callback or (lambda _c, _t, _p, _ph: None) cancel_event = cancel_event or threading.Event() - def _log(message: str) -> None: - if log_callback: - log_callback(message) + docs_dir = _ensure_docs_dir(plan.library_root) + audit_path = audit_path or os.path.join(docs_dir, "LibrarySyncAudit.json") + executed_report_path = executed_report_path or os.path.join(docs_dir, "LibrarySyncExecuted.html") + playlist_path = playlist_path or os.path.join(plan.library_root, "Playlists", "LibrarySyncExecuted.m3u") + backup_root = os.path.join(docs_dir, "Backups", "LibrarySync") + os.makedirs(backup_root, exist_ok=True) - def _progress(current: int, total: int, path: str, phase: str) -> None: - if progress_callback: - progress_callback(current, total, path, phase) + allow_all_replacements = allow_replacements is True + allowed_replacements = set(getattr(plan, "allowed_replacements", set()) or []) + if allow_replacements not in (None, False, True): + allowed_replacements.update(allow_replacements) # type: ignore[arg-type] - if cancel_event.is_set(): - _log("✘ Execution cancelled before start.") - return {"moved": 0, "errors": [], "cancelled": True} + copy_only = set(getattr(plan, "copy_only", set()) or []) moves = sorted(plan.moves.items()) total = len(moves) - summary: Dict[str, object] = {"moved": 0, "errors": [], "cancelled": False} - _log(f"Executing plan with {total} moves…") + timestamp = datetime.now(timezone.utc).isoformat() + + summary: Dict[str, object] = { + "moved": 0, + "copied": 0, + "errors": [], + "skipped": [], + "review_required": [], + "backups": [], + "cancelled": False, + "dry_run": dry_run, + } + audit_items: List[Dict[str, object]] = [] + executed_moves: Dict[str, str] = {} + + if cancel_event.is_set(): + log_callback("✘ Execution cancelled before start.") + summary["cancelled"] = True + else: + log_callback(f"Executing plan with {total} operations…") for idx, (src, dst) in enumerate(moves, start=1): if cancel_event.is_set(): summary["cancelled"] = True - _log("✘ Execution cancelled.") + log_callback("✘ Execution cancelled.") break - _progress(idx, total, src, "execute") + progress_callback(idx, total, src, "execute") + action = "copy" if src in copy_only else "move" + outcome: Dict[str, object] = { + "source": src, + "destination": dst, + "action": action, + "timestamp": timestamp, + } + + if not os.path.exists(src): + reason = "Source missing" + summary["errors"].append(f"{reason}: {src}") # type: ignore[arg-type] + outcome["status"] = "failed" + outcome["reason"] = reason + audit_items.append(outcome) + continue + + dest_exists = os.path.exists(dst) + replacement_allowed = allow_all_replacements or dst in allowed_replacements + + if dest_exists and not replacement_allowed: + reason = "Destination exists; replacement not allowed" + skip_entry = {"source": src, "destination": dst, "reason": reason} + summary["review_required"].append(skip_entry) + summary["skipped"].append(skip_entry) + outcome["status"] = "skipped" + outcome["reason"] = reason + audit_items.append(outcome) + continue + + backup_path = None + if dest_exists and replacement_allowed: + rel = os.path.relpath(dst, plan.destination_root if os.path.isdir(plan.destination_root) else plan.library_root) + backup_path = os.path.join(backup_root, rel) + if not dry_run: + try: + os.makedirs(os.path.dirname(backup_path), exist_ok=True) + shutil.copy2(dst, backup_path) + summary["backups"].append(backup_path) + except Exception as exc: # pragma: no cover - OS interaction + summary["errors"].append(f"Backup failed for {dst}: {exc}") # type: ignore[arg-type] + outcome["backup_path"] = backup_path + + if dry_run: + outcome["status"] = "dry_run" + outcome["reason"] = outcome.get("reason", "No changes written (dry run)") + executed_moves[src] = dst + audit_items.append(outcome) + continue + try: os.makedirs(os.path.dirname(dst), exist_ok=True) - shutil.move(src, dst) - summary["moved"] = int(summary["moved"]) + 1 + if dest_exists and replacement_allowed: + try: + os.remove(dst) + except FileNotFoundError: + pass + if action == "copy": + shutil.copy2(src, dst) + summary["copied"] = int(summary["copied"]) + 1 + else: + shutil.move(src, dst) + summary["moved"] = int(summary["moved"]) + 1 + executed_moves[src] = dst + outcome["status"] = "success" + if backup_path: + outcome["backup_path"] = backup_path except Exception as exc: # pragma: no cover - OS interaction - msg = f"Failed to move {src} → {dst}: {exc}" - summary.setdefault("errors", []).append(msg) - _log(msg) + msg = f"Failed to {action} {src} → {dst}: {exc}" + summary["errors"].append(msg) # type: ignore[arg-type] + outcome["status"] = "failed" + outcome["reason"] = str(exc) + log_callback(msg) + + audit_items.append(outcome) + + report_heading = "Library Sync (Executed)" if not dry_run else "Library Sync (Dry Run Execution)" + + report_skipped = [item for item in audit_items if item.get("status") == "skipped"] + report_failed = [item for item in audit_items if item.get("status") == "failed"] + report_review = [dict(rr) for rr in summary["review_required"]] # type: ignore[arg-type] + _render_execution_report( + plan, + executed_moves, + report_skipped, # type: ignore[arg-type] + report_failed, # type: ignore[arg-type] + report_review, + executed_report_path, + report_heading, + dry_run, + ) + summary["report_path"] = executed_report_path + + audit_payload = { + "executed_at": timestamp, + "dry_run": dry_run, + "plan": { + "library_root": plan.library_root, + "incoming_root": plan.incoming_root, + "destination_root": plan.destination_root, + "total_operations": total, + }, + "summary": summary, + "items": audit_items, + } + try: + with open(audit_path, "w", encoding="utf-8") as f: + json.dump(audit_payload, f, indent=2) + except Exception as exc: # pragma: no cover - OS interaction + log_callback(f"✘ Failed to write audit log: {exc}") + summary["audit_path"] = audit_path + + playlist_created = None + if create_playlist and executed_moves and not dry_run: + try: + write_playlist(sorted(executed_moves.values()), playlist_path) + playlist_created = playlist_path + except Exception as exc: # pragma: no cover - OS interaction + log_callback(f"✘ Failed to write playlist: {exc}") + summary["playlist_path"] = playlist_created if not summary.get("cancelled"): - _log("✓ Execution complete.") + log_callback("✓ Execution complete.") return summary + + +def execute_library_sync_plan( + plan: LibrarySyncPlan, + *, + log_callback: Callable[[str], None] | None = None, + progress_callback: Callable[[int, int, str, str], None] | None = None, + cancel_event: threading.Event | None = None, + dry_run: bool = False, +) -> Dict[str, object]: + """Compatibility wrapper around :func:`execute_plan`.""" + + return execute_plan( + plan, + dry_run=dry_run, + allow_replacements=False, + create_playlist=True, + log_callback=log_callback, + progress_callback=progress_callback, + cancel_event=cancel_event, + ) diff --git a/tests/test_library_sync_execute_plan.py b/tests/test_library_sync_execute_plan.py new file mode 100644 index 0000000..10ded3f --- /dev/null +++ b/tests/test_library_sync_execute_plan.py @@ -0,0 +1,136 @@ +import json +import os +import sys +import types +from pathlib import Path + +# Provide simple stubs for mutagen so the module imports +mutagen_stub = types.ModuleType("mutagen") + + +class DummyAudio: + def __init__(self): + self.tags = None + self.pictures = [] + + +def File(*_a, **_k): + return DummyAudio() + + +mutagen_stub.File = File + +id3_stub = types.ModuleType("id3") + + +class DummyID3Error(Exception): + pass + + +id3_stub.ID3NoHeaderError = DummyID3Error +mutagen_stub.id3 = id3_stub +sys.modules.setdefault("mutagen", mutagen_stub) +sys.modules.setdefault("mutagen.id3", id3_stub) +acoustid_stub = types.SimpleNamespace(fingerprint_file=lambda *_a, **_k: (None, None)) +sys.modules.setdefault("acoustid", acoustid_stub) + +from library_sync import LibrarySyncPlan, execute_plan + + +def _build_plan(tmp_path, filename="track.mp3"): + library_root = tmp_path / "library" + incoming_root = tmp_path / "incoming" + dest_root = library_root / "Music" + dest_root.mkdir(parents=True) + incoming_root.mkdir() + library_root.mkdir(exist_ok=True) + + src = incoming_root / filename + src.write_text("new audio") + dest = dest_root / "Artist" / filename + + plan = LibrarySyncPlan( + library_root=str(library_root), + incoming_root=str(incoming_root), + destination_root=str(dest_root), + moves={str(src): str(dest)}, + tag_index={str(dest): {"leftover_tags": []}}, + decision_log=[], + ) + return plan, src, dest + + +def test_execute_plan_writes_audit_and_report(tmp_path): + plan, src, dest = _build_plan(tmp_path) + audit_path = tmp_path / "audit.json" + report_path = tmp_path / "executed.html" + playlist_path = tmp_path / "executed.m3u" + + summary = execute_plan( + plan, + audit_path=str(audit_path), + executed_report_path=str(report_path), + playlist_path=str(playlist_path), + log_callback=lambda _m: None, + ) + + assert summary["moved"] == 1 + assert summary["audit_path"] == str(audit_path) + assert dest.exists() + assert not src.exists() + + payload = json.loads(audit_path.read_text()) + assert payload["items"][0]["status"] == "success" + assert payload["summary"]["moved"] == 1 + + content = report_path.read_text() + assert "Library Sync (Executed" in content + assert dest.name in content + + assert summary["playlist_path"] == str(playlist_path) + assert playlist_path.exists() + assert dest.name in playlist_path.read_text() + + +def test_execute_plan_handles_replacements_and_backups(tmp_path): + plan, src, dest = _build_plan(tmp_path, "replace.mp3") + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_text("old audio") + + audit_path = tmp_path / "audit_skip.json" + report_path = tmp_path / "report_skip.html" + + summary = execute_plan( + plan, + audit_path=str(audit_path), + executed_report_path=str(report_path), + create_playlist=False, + log_callback=lambda _m: None, + ) + + assert summary["moved"] == 0 + assert dest.read_text() == "old audio" + assert src.exists() + payload = json.loads(audit_path.read_text()) + assert payload["items"][0]["status"] == "skipped" + assert payload["summary"]["review_required"] + + plan.allowed_replacements.add(str(dest)) + audit_path2 = tmp_path / "audit_replace.json" + report_path2 = tmp_path / "report_replace.html" + + summary2 = execute_plan( + plan, + audit_path=str(audit_path2), + executed_report_path=str(report_path2), + create_playlist=False, + log_callback=lambda _m: None, + ) + + assert summary2["moved"] == 1 + assert not src.exists() + assert dest.read_text() == "new audio" + assert summary2["backups"] + backup_path = summary2["backups"][0] + assert backup_path and os.path.exists(backup_path) + assert Path(backup_path).read_text() == "old audio" From 6a84cd93ac2531c402406232d8032ccd36f602cb Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sat, 20 Dec 2025 14:09:16 -0500 Subject: [PATCH 138/606] Prevent library sync tree selection recursion --- library_sync_review.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/library_sync_review.py b/library_sync_review.py index 6f41b18..e349c7c 100644 --- a/library_sync_review.py +++ b/library_sync_review.py @@ -207,6 +207,7 @@ def __init__( self.report_version_var = tk.StringVar(value=str(self.session.exported_report_version)) self.scan_state_var = tk.StringVar(value=self._describe_state(self.session.scan_state)) self._on_close_callback = on_close + self._selection_syncing = False self.library_var.trace_add("write", lambda *_: self._invalidate_plan("Library folder changed")) self.incoming_var.trace_add("write", lambda *_: self._invalidate_plan("Incoming folder changed")) @@ -998,17 +999,24 @@ def _on_incoming_select(self, _event=None) -> None: sel = self.incoming_tree.selection() if not sel: return + if self._selection_syncing: + return + self._selection_syncing = True incoming_id = sel[0] match = self.match_by_incoming.get(incoming_id) if match and match.existing: self.existing_tree.selection_set(match.existing.track_id) self.existing_tree.see(match.existing.track_id) + self._selection_syncing = False self._update_inspector(incoming_id=incoming_id) def _on_existing_select(self, _event=None) -> None: sel = self.existing_tree.selection() if not sel: return + if self._selection_syncing: + return + self._selection_syncing = True existing_id = sel[0] linked = self.match_by_existing.get(existing_id, []) if linked: @@ -1018,6 +1026,7 @@ def _on_existing_select(self, _event=None) -> None: self._update_inspector(incoming_id=target) else: self._update_inspector(existing_id=existing_id) + self._selection_syncing = False def _update_inspector(self, incoming_id: str | None = None, existing_id: str | None = None) -> None: res: MatchResult | None = None From 01e04b5ea170ba851677ee1d23cf8f64a5c80d53 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sat, 20 Dec 2025 14:10:59 -0500 Subject: [PATCH 139/606] Revert "Prevent library sync tree selection recursion" --- library_sync_review.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/library_sync_review.py b/library_sync_review.py index e349c7c..6f41b18 100644 --- a/library_sync_review.py +++ b/library_sync_review.py @@ -207,7 +207,6 @@ def __init__( self.report_version_var = tk.StringVar(value=str(self.session.exported_report_version)) self.scan_state_var = tk.StringVar(value=self._describe_state(self.session.scan_state)) self._on_close_callback = on_close - self._selection_syncing = False self.library_var.trace_add("write", lambda *_: self._invalidate_plan("Library folder changed")) self.incoming_var.trace_add("write", lambda *_: self._invalidate_plan("Incoming folder changed")) @@ -999,24 +998,17 @@ def _on_incoming_select(self, _event=None) -> None: sel = self.incoming_tree.selection() if not sel: return - if self._selection_syncing: - return - self._selection_syncing = True incoming_id = sel[0] match = self.match_by_incoming.get(incoming_id) if match and match.existing: self.existing_tree.selection_set(match.existing.track_id) self.existing_tree.see(match.existing.track_id) - self._selection_syncing = False self._update_inspector(incoming_id=incoming_id) def _on_existing_select(self, _event=None) -> None: sel = self.existing_tree.selection() if not sel: return - if self._selection_syncing: - return - self._selection_syncing = True existing_id = sel[0] linked = self.match_by_existing.get(existing_id, []) if linked: @@ -1026,7 +1018,6 @@ def _on_existing_select(self, _event=None) -> None: self._update_inspector(incoming_id=target) else: self._update_inspector(existing_id=existing_id) - self._selection_syncing = False def _update_inspector(self, incoming_id: str | None = None, existing_id: str | None = None) -> None: res: MatchResult | None = None From 2313fab8bf678e55d4c4c543fae76f00a555d1b1 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sat, 20 Dec 2025 14:29:53 -0500 Subject: [PATCH 140/606] Remove cross-selection in library sync inspector --- library_sync_review.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/library_sync_review.py b/library_sync_review.py index 6f41b18..1af6b33 100644 --- a/library_sync_review.py +++ b/library_sync_review.py @@ -999,10 +999,6 @@ def _on_incoming_select(self, _event=None) -> None: if not sel: return incoming_id = sel[0] - match = self.match_by_incoming.get(incoming_id) - if match and match.existing: - self.existing_tree.selection_set(match.existing.track_id) - self.existing_tree.see(match.existing.track_id) self._update_inspector(incoming_id=incoming_id) def _on_existing_select(self, _event=None) -> None: @@ -1013,8 +1009,6 @@ def _on_existing_select(self, _event=None) -> None: linked = self.match_by_existing.get(existing_id, []) if linked: target = linked[0].incoming.track_id - self.incoming_tree.selection_set(target) - self.incoming_tree.see(target) self._update_inspector(incoming_id=target) else: self._update_inspector(existing_id=existing_id) From e056c2c81fd792d8e30366309ad8454a7cdba546 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sat, 20 Dec 2025 16:13:06 -0500 Subject: [PATCH 141/606] Annotate library sync preview decisions --- library_sync.py | 137 +++++++++++++++++- .../indexer_engine/music_indexer_api.py | 68 ++++++++- music_indexer_api.py | 68 ++++++++- tests/test_library_sync_plan_preview.py | 30 +++- 4 files changed, 289 insertions(+), 14 deletions(-) diff --git a/library_sync.py b/library_sync.py index 7a3a9d5..e13ae8b 100644 --- a/library_sync.py +++ b/library_sync.py @@ -1,6 +1,7 @@ """Helpers for comparing an existing library to an incoming folder.""" from __future__ import annotations +import filecmp import hashlib import json import logging @@ -96,6 +97,16 @@ class MatchStatus(str, Enum): LOW_CONFIDENCE = "low_confidence" +class ExecutionDecision(str, Enum): + """Execution outcome for a planned incoming track.""" + + COPY = "COPY" + REPLACE = "REPLACE" + SKIP_DUPLICATE = "SKIP_DUPLICATE" + SKIP_KEEP_EXISTING = "SKIP_KEEP_EXISTING" + REVIEW_REQUIRED = "REVIEW_REQUIRED" + + @dataclass class TrackRecord: """Normalized representation of a scanned track.""" @@ -211,6 +222,26 @@ def to_dict(self) -> Dict[str, object]: } +@dataclass +class PlannedItem: + """Planned movement decision for a single incoming track.""" + + source: str + destination: str + decision: ExecutionDecision + reason: str | None = None + action: str = "move" + + def to_dict(self) -> Dict[str, str]: + return { + "source": self.source, + "destination": self.destination, + "decision": self.decision.value, + "reason": self.reason or "", + "action": self.action, + } + + def _fingerprint_signature(fp: str | None) -> str | None: if not fp: return None @@ -673,25 +704,83 @@ class LibrarySyncPlan: decision_log: List[str] copy_only: set[str] = field(default_factory=set) allowed_replacements: set[str] = field(default_factory=set) + items: List[PlannedItem] = field(default_factory=list) def planned_moves(self) -> Dict[str, str]: """Return a copy of the planned move mapping for execution.""" + if self.items: + return {item.source: item.destination for item in self.items} return dict(self.moves) def render_preview(self, output_html_path: str) -> str: """Generate a dry-run HTML preview from this plan.""" heading = "Library Sync (Dry Run Preview)" + preview_items = [item.to_dict() for item in self.items] if self.items else None idx.render_dry_run_html_from_plan( self.destination_root, output_html_path, self.moves, self.tag_index, + plan_items=preview_items, heading_text=heading, title_prefix=heading, ) return output_html_path +def _files_match(path_a: str, path_b: str) -> bool: + try: + return filecmp.cmp(path_a, path_b, shallow=False) + except Exception: + return False + + +def _compute_plan_items( + moves: Dict[str, str], + *, + destination_root: str, + copy_only: Iterable[str] | None = None, + allow_all_replacements: bool = False, + allowed_replacements: Iterable[str] | None = None, +) -> List[PlannedItem]: + """Attach deterministic execution decisions to each planned move.""" + + copy_only_set = set(copy_only or []) + allowed_replacements_set = set(allowed_replacements or []) + items: List[PlannedItem] = [] + + for src, dst in sorted(moves.items(), key=lambda item: item[1]): + action = "copy" if src in copy_only_set else "move" + dest_exists = os.path.exists(dst) + replacement_allowed = allow_all_replacements or dst in allowed_replacements_set + + if dest_exists and not replacement_allowed: + if _files_match(src, dst): + decision = ExecutionDecision.SKIP_DUPLICATE + reason = "Exact duplicate already in library" + else: + decision = ExecutionDecision.SKIP_KEEP_EXISTING + reason = "Destination exists; keeping library copy" + elif dest_exists and replacement_allowed: + decision = ExecutionDecision.REPLACE + reason = "Replacing existing file" + else: + decision = ExecutionDecision.COPY + reason = "Plan will transfer file" + + items.append( + PlannedItem( + source=src, + destination=dst, + decision=decision, + reason=reason, + action=action, + ) + ) + + return items + + def compute_library_sync_plan( library_root: str, incoming_folder: str, @@ -746,6 +835,14 @@ def _remap_path(path: str) -> str: remapped_moves = {_normalize_path(src): _remap_path(dst) for src, dst in moves.items()} remapped_tag_index = {_remap_path(dst): data for dst, data in tag_index.items()} + plan_items = _compute_plan_items( + remapped_moves, + destination_root=_normalize_path(destination_root), + copy_only=None, + allow_all_replacements=False, + allowed_replacements=None, + ) + return LibrarySyncPlan( library_root=_normalize_path(library_root), incoming_root=_normalize_path(incoming_folder), @@ -753,6 +850,7 @@ def _remap_path(path: str) -> str: moves=remapped_moves, tag_index=remapped_tag_index, decision_log=decision_log, + items=plan_items, ) @@ -889,8 +987,16 @@ def execute_plan( copy_only = set(getattr(plan, "copy_only", set()) or []) - moves = sorted(plan.moves.items()) - total = len(moves) + plan_items = _compute_plan_items( + plan.planned_moves(), + destination_root=plan.destination_root, + copy_only=copy_only, + allow_all_replacements=allow_all_replacements, + allowed_replacements=allowed_replacements, + ) + plan.items = plan_items + + total = len(plan_items) timestamp = datetime.now(timezone.utc).isoformat() summary: Dict[str, object] = { @@ -912,14 +1018,16 @@ def execute_plan( else: log_callback(f"Executing plan with {total} operations…") - for idx, (src, dst) in enumerate(moves, start=1): + for idx, item in enumerate(plan_items, start=1): if cancel_event.is_set(): summary["cancelled"] = True log_callback("✘ Execution cancelled.") break + src = item.source + dst = item.destination progress_callback(idx, total, src, "execute") - action = "copy" if src in copy_only else "move" + action = item.action or ("copy" if src in copy_only else "move") outcome: Dict[str, object] = { "source": src, "destination": dst, @@ -927,6 +1035,25 @@ def execute_plan( "timestamp": timestamp, } + if item.decision in ( + ExecutionDecision.SKIP_DUPLICATE, + ExecutionDecision.SKIP_KEEP_EXISTING, + ExecutionDecision.REVIEW_REQUIRED, + ): + reason = item.reason or "Skipped by plan decision" + skip_entry = {"source": src, "destination": dst, "reason": reason} + summary["skipped"].append(skip_entry) + if item.decision in ( + ExecutionDecision.SKIP_DUPLICATE, + ExecutionDecision.SKIP_KEEP_EXISTING, + ExecutionDecision.REVIEW_REQUIRED, + ): + summary["review_required"].append(skip_entry) + outcome["status"] = "skipped" + outcome["reason"] = reason + audit_items.append(outcome) + continue + if not os.path.exists(src): reason = "Source missing" summary["errors"].append(f"{reason}: {src}") # type: ignore[arg-type] @@ -936,7 +1063,7 @@ def execute_plan( continue dest_exists = os.path.exists(dst) - replacement_allowed = allow_all_replacements or dst in allowed_replacements + replacement_allowed = item.decision == ExecutionDecision.REPLACE if dest_exists and not replacement_allowed: reason = "Destination exists; replacement not allowed" diff --git a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py index ecb42fd..1c7d30a 100644 --- a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py +++ b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py @@ -762,6 +762,7 @@ def render_dry_run_html_from_plan( output_html_path, moves, tag_index, + plan_items=None, *, heading_text="Phase A – Exact Metadata", title_prefix="Music Index (Dry Run)", @@ -769,11 +770,63 @@ def render_dry_run_html_from_plan( """Render a dry-run preview from an already computed move plan.""" coord = DryRunCoordinator() + def _decision_annotation(decision: str): + mapping = { + "SKIP_DUPLICATE": ("(duplicate)", "decision-skip"), + "SKIP_KEEP_EXISTING": ("(kept existing)", "decision-skip"), + "REVIEW_REQUIRED": ("(review required)", "decision-review"), + "REPLACE": ("(replace)", "decision-replace"), + "COPY": ("(copy)", "decision-copy"), + } + return mapping.get(decision.upper()) + + def _render_legend(decisions): + if not decisions: + return "" + legend_lines = ["
      Legend:"] + descriptions = { + "SKIP_DUPLICATE": "Identical file already exists; will be skipped.", + "SKIP_KEEP_EXISTING": "A different file is already present; keeping existing.", + "REVIEW_REQUIRED": "Needs manual approval before moving.", + "REPLACE": "Existing file will be replaced.", + "COPY": "Incoming file will be copied/moved.", + } + for key in ["SKIP_DUPLICATE", "SKIP_KEEP_EXISTING", "REVIEW_REQUIRED", "REPLACE", "COPY"]: + if key not in decisions: + continue + annotation = _decision_annotation(key) + if not annotation: + continue + label, css = annotation + desc = descriptions.get(key, "") + legend_lines.append( + f"
      {sanitize(label)} " + f"{sanitize(desc)}
      " + ) + legend_lines.append("
      ") + return "\n".join(legend_lines) + def build_exact_metadata_section() -> str: - lines = [f"

      {heading_text}

      ", "
      "]
      +        lines = [f"

      {heading_text}

      "] + decision_map = {} + decision_keys = set() + if plan_items: + for item in plan_items: + dst = item.get("destination") + decision = (item.get("decision") or "").upper() + annotation = _decision_annotation(decision) + if dst and annotation: + decision_map[dst] = annotation + decision_keys.add(decision) + + legend_html = _render_legend(decision_keys) + if legend_html: + lines.append(legend_html) + lines.append("
      ")
               lines.append(f"{sanitize(os.path.basename(root_path))}/")
               tree_nodes = set()
      -        for new_path in moves.values():
      +        target_paths = [item.get("destination") for item in plan_items if item.get("destination")] if plan_items else list(moves.values())
      +        for new_path in target_paths:
                   parts = os.path.relpath(new_path, root_path).split(os.sep)
                   for i in range(1, len(parts) + 1):
                       subtree = os.path.join(root_path, *parts[:i])
      @@ -789,7 +842,14 @@ def build_exact_metadata_section() -> str:
                       fname = os.path.basename(node)
                       leftover = tag_index.get(node, {}).get("leftover_tags", [])
                       tags_str = ", ".join(leftover) if leftover else ""
      -                line = f"{indent}- {sanitize(fname)}"
      +                annotation = decision_map.get(node)
      +                classes = ["song"]
      +                label = ""
      +                if annotation:
      +                    label_text, css_class = annotation
      +                    classes.append(css_class)
      +                    label = f" {sanitize(label_text)}"
      +                line = f"{indent}- {sanitize(fname)}{label}"
                       if tags_str:
                           line += f"  [{sanitize(tags_str)}]"
                       lines.append(line)
      @@ -805,7 +865,7 @@ def build_exact_metadata_section() -> str:
               out.write(
                   "\n\n\n  \n  "
                   f"{title_prefix} – {sanitize(os.path.basename(root_path))}\n  "
      -            "\n\n\n"
      +            "\n\n\n"
               )
               out.write(html_body)
               if html_body and not html_body.endswith("\n"):
      diff --git a/music_indexer_api.py b/music_indexer_api.py
      index ecb42fd..1c7d30a 100644
      --- a/music_indexer_api.py
      +++ b/music_indexer_api.py
      @@ -762,6 +762,7 @@ def render_dry_run_html_from_plan(
           output_html_path,
           moves,
           tag_index,
      +    plan_items=None,
           *,
           heading_text="Phase A – Exact Metadata",
           title_prefix="Music Index (Dry Run)",
      @@ -769,11 +770,63 @@ def render_dry_run_html_from_plan(
           """Render a dry-run preview from an already computed move plan."""
           coord = DryRunCoordinator()
       
      +    def _decision_annotation(decision: str):
      +        mapping = {
      +            "SKIP_DUPLICATE": ("(duplicate)", "decision-skip"),
      +            "SKIP_KEEP_EXISTING": ("(kept existing)", "decision-skip"),
      +            "REVIEW_REQUIRED": ("(review required)", "decision-review"),
      +            "REPLACE": ("(replace)", "decision-replace"),
      +            "COPY": ("(copy)", "decision-copy"),
      +        }
      +        return mapping.get(decision.upper())
      +
      +    def _render_legend(decisions):
      +        if not decisions:
      +            return ""
      +        legend_lines = ["
      Legend:"] + descriptions = { + "SKIP_DUPLICATE": "Identical file already exists; will be skipped.", + "SKIP_KEEP_EXISTING": "A different file is already present; keeping existing.", + "REVIEW_REQUIRED": "Needs manual approval before moving.", + "REPLACE": "Existing file will be replaced.", + "COPY": "Incoming file will be copied/moved.", + } + for key in ["SKIP_DUPLICATE", "SKIP_KEEP_EXISTING", "REVIEW_REQUIRED", "REPLACE", "COPY"]: + if key not in decisions: + continue + annotation = _decision_annotation(key) + if not annotation: + continue + label, css = annotation + desc = descriptions.get(key, "") + legend_lines.append( + f"
      {sanitize(label)} " + f"{sanitize(desc)}
      " + ) + legend_lines.append("
      ") + return "\n".join(legend_lines) + def build_exact_metadata_section() -> str: - lines = [f"

      {heading_text}

      ", "
      "]
      +        lines = [f"

      {heading_text}

      "] + decision_map = {} + decision_keys = set() + if plan_items: + for item in plan_items: + dst = item.get("destination") + decision = (item.get("decision") or "").upper() + annotation = _decision_annotation(decision) + if dst and annotation: + decision_map[dst] = annotation + decision_keys.add(decision) + + legend_html = _render_legend(decision_keys) + if legend_html: + lines.append(legend_html) + lines.append("
      ")
               lines.append(f"{sanitize(os.path.basename(root_path))}/")
               tree_nodes = set()
      -        for new_path in moves.values():
      +        target_paths = [item.get("destination") for item in plan_items if item.get("destination")] if plan_items else list(moves.values())
      +        for new_path in target_paths:
                   parts = os.path.relpath(new_path, root_path).split(os.sep)
                   for i in range(1, len(parts) + 1):
                       subtree = os.path.join(root_path, *parts[:i])
      @@ -789,7 +842,14 @@ def build_exact_metadata_section() -> str:
                       fname = os.path.basename(node)
                       leftover = tag_index.get(node, {}).get("leftover_tags", [])
                       tags_str = ", ".join(leftover) if leftover else ""
      -                line = f"{indent}- {sanitize(fname)}"
      +                annotation = decision_map.get(node)
      +                classes = ["song"]
      +                label = ""
      +                if annotation:
      +                    label_text, css_class = annotation
      +                    classes.append(css_class)
      +                    label = f" {sanitize(label_text)}"
      +                line = f"{indent}- {sanitize(fname)}{label}"
                       if tags_str:
                           line += f"  [{sanitize(tags_str)}]"
                       lines.append(line)
      @@ -805,7 +865,7 @@ def build_exact_metadata_section() -> str:
               out.write(
                   "\n\n\n  \n  "
                   f"{title_prefix} – {sanitize(os.path.basename(root_path))}\n  "
      -            "\n\n\n"
      +            "\n\n\n"
               )
               out.write(html_body)
               if html_body and not html_body.endswith("\n"):
      diff --git a/tests/test_library_sync_plan_preview.py b/tests/test_library_sync_plan_preview.py
      index 1168f87..3669cfa 100644
      --- a/tests/test_library_sync_plan_preview.py
      +++ b/tests/test_library_sync_plan_preview.py
      @@ -33,7 +33,7 @@ class DummyID3Error(Exception):
       acoustid_stub = types.SimpleNamespace(fingerprint_file=lambda *_a, **_k: (None, None))
       sys.modules.setdefault("acoustid", acoustid_stub)
       
      -from library_sync import compute_library_sync_plan, execute_library_sync_plan
      +from library_sync import LibrarySyncPlan, _compute_plan_items, compute_library_sync_plan, execute_library_sync_plan
       
       
       def test_library_sync_plan_preview_reuses_plan(tmp_path):
      @@ -102,3 +102,31 @@ def test_execute_library_sync_plan_honors_cancellation(tmp_path):
       
           assert summary["cancelled"]
           assert incoming_song.exists()
      +
      +
      +def test_preview_marks_duplicate_items(tmp_path):
      +    library_root = tmp_path / "library"
      +    incoming_root = tmp_path / "incoming"
      +    dest_root = library_root / "Music"
      +    dest_root.mkdir(parents=True)
      +    incoming_root.mkdir()
      +
      +    src = incoming_root / "dup.mp3"
      +    dst = dest_root / "dup.mp3"
      +    src.write_text("audio")
      +    dst.write_text("audio")
      +
      +    plan = LibrarySyncPlan(
      +        library_root=str(library_root),
      +        incoming_root=str(incoming_root),
      +        destination_root=str(dest_root),
      +        moves={str(src): str(dst)},
      +        tag_index={str(dst): {"leftover_tags": []}},
      +        decision_log=[],
      +    )
      +    plan.items = _compute_plan_items(plan.moves, destination_root=str(dest_root))
      +    html_path = tmp_path / "preview.html"
      +    plan.render_preview(str(html_path))
      +    content = html_path.read_text()
      +
      +    assert "(duplicate)" in content
      
      From 8ce8f2c584174a845007d9d63ab372d69afeecfb Mon Sep 17 00:00:00 2001
      From: Greensand321 
      Date: Sat, 20 Dec 2025 16:27:15 -0500
      Subject: [PATCH 142/606] Update library sync copy label to move
      
      ---
       .../indexer_engine/music_indexer_api.py                       | 4 ++--
       music_indexer_api.py                                          | 4 ++--
       2 files changed, 4 insertions(+), 4 deletions(-)
      
      diff --git a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py
      index 1c7d30a..ae2d511 100644
      --- a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py
      +++ b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py
      @@ -776,7 +776,7 @@ def _decision_annotation(decision: str):
                   "SKIP_KEEP_EXISTING": ("(kept existing)", "decision-skip"),
                   "REVIEW_REQUIRED": ("(review required)", "decision-review"),
                   "REPLACE": ("(replace)", "decision-replace"),
      -            "COPY": ("(copy)", "decision-copy"),
      +            "COPY": ("(move)", "decision-copy"),
               }
               return mapping.get(decision.upper())
       
      @@ -789,7 +789,7 @@ def _render_legend(decisions):
                   "SKIP_KEEP_EXISTING": "A different file is already present; keeping existing.",
                   "REVIEW_REQUIRED": "Needs manual approval before moving.",
                   "REPLACE": "Existing file will be replaced.",
      -            "COPY": "Incoming file will be copied/moved.",
      +            "COPY": "Incoming file will be moved.",
               }
               for key in ["SKIP_DUPLICATE", "SKIP_KEEP_EXISTING", "REVIEW_REQUIRED", "REPLACE", "COPY"]:
                   if key not in decisions:
      diff --git a/music_indexer_api.py b/music_indexer_api.py
      index 1c7d30a..ae2d511 100644
      --- a/music_indexer_api.py
      +++ b/music_indexer_api.py
      @@ -776,7 +776,7 @@ def _decision_annotation(decision: str):
                   "SKIP_KEEP_EXISTING": ("(kept existing)", "decision-skip"),
                   "REVIEW_REQUIRED": ("(review required)", "decision-review"),
                   "REPLACE": ("(replace)", "decision-replace"),
      -            "COPY": ("(copy)", "decision-copy"),
      +            "COPY": ("(move)", "decision-copy"),
               }
               return mapping.get(decision.upper())
       
      @@ -789,7 +789,7 @@ def _render_legend(decisions):
                   "SKIP_KEEP_EXISTING": "A different file is already present; keeping existing.",
                   "REVIEW_REQUIRED": "Needs manual approval before moving.",
                   "REPLACE": "Existing file will be replaced.",
      -            "COPY": "Incoming file will be copied/moved.",
      +            "COPY": "Incoming file will be moved.",
               }
               for key in ["SKIP_DUPLICATE", "SKIP_KEEP_EXISTING", "REVIEW_REQUIRED", "REPLACE", "COPY"]:
                   if key not in decisions:
      
      From c51d7589b18cd1b493e415341ee3e234d674f1ad Mon Sep 17 00:00:00 2001
      From: Greensand321 
      Date: Sat, 20 Dec 2025 16:53:03 -0500
      Subject: [PATCH 143/606] Add transfer mode toggle and playlist output
      
      ---
       library_sync.py                               |  24 +++-
       .../indexer_engine/music_indexer_api.py       |  25 ++--
       library_sync_review.py                        | 119 ++++++++++++++++--
       music_indexer_api.py                          |  25 ++--
       4 files changed, 166 insertions(+), 27 deletions(-)
      
      diff --git a/library_sync.py b/library_sync.py
      index e13ae8b..4ce956f 100644
      --- a/library_sync.py
      +++ b/library_sync.py
      @@ -705,6 +705,7 @@ class LibrarySyncPlan:
           copy_only: set[str] = field(default_factory=set)
           allowed_replacements: set[str] = field(default_factory=set)
           items: List[PlannedItem] = field(default_factory=list)
      +    transfer_mode: str = "move"
       
           def planned_moves(self) -> Dict[str, str]:
               """Return a copy of the planned move mapping for execution."""
      @@ -742,15 +743,17 @@ def _compute_plan_items(
           copy_only: Iterable[str] | None = None,
           allow_all_replacements: bool = False,
           allowed_replacements: Iterable[str] | None = None,
      +    transfer_mode: str = "move",
       ) -> List[PlannedItem]:
           """Attach deterministic execution decisions to each planned move."""
       
           copy_only_set = set(copy_only or [])
           allowed_replacements_set = set(allowed_replacements or [])
      +    normalized_transfer_mode = "copy" if str(transfer_mode).lower() == "copy" else "move"
           items: List[PlannedItem] = []
       
           for src, dst in sorted(moves.items(), key=lambda item: item[1]):
      -        action = "copy" if src in copy_only_set else "move"
      +        action = "copy" if src in copy_only_set else normalized_transfer_mode
               dest_exists = os.path.exists(dst)
               replacement_allowed = allow_all_replacements or dst in allowed_replacements_set
       
      @@ -790,6 +793,7 @@ def compute_library_sync_plan(
           flush_cache: bool = False,
           max_workers: int | None = None,
           cancel_event: threading.Event | None = None,
      +    transfer_mode: str = "move",
       ) -> LibrarySyncPlan:
           """Compute a deterministic move/route plan for Library Sync."""
           cancel_event = cancel_event or threading.Event()
      @@ -841,6 +845,7 @@ def _remap_path(path: str) -> str:
               copy_only=None,
               allow_all_replacements=False,
               allowed_replacements=None,
      +        transfer_mode=transfer_mode,
           )
       
           return LibrarySyncPlan(
      @@ -851,6 +856,7 @@ def _remap_path(path: str) -> str:
               tag_index=remapped_tag_index,
               decision_log=decision_log,
               items=plan_items,
      +        transfer_mode="copy" if str(transfer_mode).lower() == "copy" else "move",
           )
       
       
      @@ -864,6 +870,7 @@ def build_library_sync_preview(
           flush_cache: bool = False,
           max_workers: int | None = None,
           cancel_event: threading.Event | None = None,
      +    transfer_mode: str = "move",
       ) -> LibrarySyncPlan:
           """Compute a Library Sync plan and write the dry-run preview."""
           plan = compute_library_sync_plan(
      @@ -874,6 +881,7 @@ def build_library_sync_preview(
               flush_cache=flush_cache,
               max_workers=max_workers,
               cancel_event=cancel_event,
      +        transfer_mode=transfer_mode,
           )
           plan.render_preview(output_html_path)
           return plan
      @@ -976,7 +984,11 @@ def execute_plan(
           docs_dir = _ensure_docs_dir(plan.library_root)
           audit_path = audit_path or os.path.join(docs_dir, "LibrarySyncAudit.json")
           executed_report_path = executed_report_path or os.path.join(docs_dir, "LibrarySyncExecuted.html")
      -    playlist_path = playlist_path or os.path.join(plan.library_root, "Playlists", "LibrarySyncExecuted.m3u")
      +    playlist_root = os.path.join(plan.library_root, "Playlists")
      +    os.makedirs(playlist_root, exist_ok=True)
      +    if playlist_path is None:
      +        playlist_stamp = datetime.now().strftime("%Y-%m-%d_%H%M")
      +        playlist_path = os.path.join(playlist_root, f"LibrarySync_Added_{playlist_stamp}.m3u8")
           backup_root = os.path.join(docs_dir, "Backups", "LibrarySync")
           os.makedirs(backup_root, exist_ok=True)
       
      @@ -993,6 +1005,7 @@ def execute_plan(
               copy_only=copy_only,
               allow_all_replacements=allow_all_replacements,
               allowed_replacements=allowed_replacements,
      +        transfer_mode=getattr(plan, "transfer_mode", "move"),
           )
           plan.items = plan_items
       
      @@ -1002,6 +1015,7 @@ def execute_plan(
           summary: Dict[str, object] = {
               "moved": 0,
               "copied": 0,
      +        "transferred": 0,
               "errors": [],
               "skipped": [],
               "review_required": [],
      @@ -1108,6 +1122,7 @@ def execute_plan(
                   else:
                       shutil.move(src, dst)
                       summary["moved"] = int(summary["moved"]) + 1
      +            summary["transferred"] = int(summary["transferred"]) + 1
                   executed_moves[src] = dst
                   outcome["status"] = "success"
                   if backup_path:
      @@ -1121,6 +1136,8 @@ def execute_plan(
       
               audit_items.append(outcome)
       
      +    summary["transferred"] = int(summary.get("moved", 0)) + int(summary.get("copied", 0))
      +
           report_heading = "Library Sync (Executed)" if not dry_run else "Library Sync (Dry Run Execution)"
       
           report_skipped = [item for item in audit_items if item.get("status") == "skipped"]
      @@ -1178,6 +1195,7 @@ def execute_library_sync_plan(
           progress_callback: Callable[[int, int, str, str], None] | None = None,
           cancel_event: threading.Event | None = None,
           dry_run: bool = False,
      +    create_playlist: bool = True,
       ) -> Dict[str, object]:
           """Compatibility wrapper around :func:`execute_plan`."""
       
      @@ -1185,7 +1203,7 @@ def execute_library_sync_plan(
               plan,
               dry_run=dry_run,
               allow_replacements=False,
      -        create_playlist=True,
      +        create_playlist=create_playlist,
               log_callback=log_callback,
               progress_callback=progress_callback,
               cancel_event=cancel_event,
      diff --git a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py
      index ae2d511..7a6131c 100644
      --- a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py
      +++ b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py
      @@ -770,17 +770,19 @@ def render_dry_run_html_from_plan(
           """Render a dry-run preview from an already computed move plan."""
           coord = DryRunCoordinator()
       
      -    def _decision_annotation(decision: str):
      +    def _decision_annotation(decision: str, action: str | None = None):
      +        normalized_action = (action or "").lower()
      +        action_label = "(copy)" if normalized_action == "copy" else "(move)"
               mapping = {
                   "SKIP_DUPLICATE": ("(duplicate)", "decision-skip"),
                   "SKIP_KEEP_EXISTING": ("(kept existing)", "decision-skip"),
                   "REVIEW_REQUIRED": ("(review required)", "decision-review"),
                   "REPLACE": ("(replace)", "decision-replace"),
      -            "COPY": ("(move)", "decision-copy"),
      +            "COPY": (action_label, "decision-copy" if normalized_action == "copy" else "decision-move"),
               }
               return mapping.get(decision.upper())
       
      -    def _render_legend(decisions):
      +    def _render_legend(decisions, action_map=None):
               if not decisions:
                   return ""
               legend_lines = ["
      Legend:"] @@ -789,12 +791,15 @@ def _render_legend(decisions): "SKIP_KEEP_EXISTING": "A different file is already present; keeping existing.", "REVIEW_REQUIRED": "Needs manual approval before moving.", "REPLACE": "Existing file will be replaced.", - "COPY": "Incoming file will be moved.", + "COPY": "Incoming file will be transferred.", } for key in ["SKIP_DUPLICATE", "SKIP_KEEP_EXISTING", "REVIEW_REQUIRED", "REPLACE", "COPY"]: if key not in decisions: continue - annotation = _decision_annotation(key) + action = None + if action_map and key in action_map and action_map[key]: + action = sorted(action_map[key])[0] + annotation = _decision_annotation(key, action or "move") if not annotation: continue label, css = annotation @@ -810,16 +815,20 @@ def build_exact_metadata_section() -> str: lines = [f"

      {heading_text}

      "] decision_map = {} decision_keys = set() + decision_actions = {} if plan_items: for item in plan_items: dst = item.get("destination") decision = (item.get("decision") or "").upper() - annotation = _decision_annotation(decision) + annotation = _decision_annotation(decision, item.get("action")) if dst and annotation: decision_map[dst] = annotation decision_keys.add(decision) + action_val = (item.get("action") or "").lower() + if action_val: + decision_actions.setdefault(decision, set()).add(action_val) - legend_html = _render_legend(decision_keys) + legend_html = _render_legend(decision_keys, decision_actions if decision_actions else None) if legend_html: lines.append(legend_html) lines.append("
      ")
      @@ -865,7 +874,7 @@ def build_exact_metadata_section() -> str:
               out.write(
                   "\n\n\n  \n  "
                   f"{title_prefix} – {sanitize(os.path.basename(root_path))}\n  "
      -            "\n\n\n"
      +            "\n\n\n"
               )
               out.write(html_body)
               if html_body and not html_body.endswith("\n"):
      diff --git a/library_sync_review.py b/library_sync_review.py
      index 1af6b33..25773fe 100644
      --- a/library_sync_review.py
      +++ b/library_sync_review.py
      @@ -70,6 +70,8 @@ class ScanSession:
           scan_config: ScanConfig = field(default_factory=ScanConfig)
           scan_state: ScanState = ScanState.IDLE
           exported_report_version: int = DEFAULT_REPORT_VERSION
      +    transfer_mode: str = "copy"
      +    output_playlist: bool = False
       
           def to_dict(self) -> Dict[str, Any]:
               return {
      @@ -78,6 +80,8 @@ def to_dict(self) -> Dict[str, Any]:
                   "scan_config": self.scan_config.to_dict(),
                   "scan_state": self.scan_state.value,
                   "exported_report_version": int(self.exported_report_version),
      +            "transfer_mode": self.transfer_mode,
      +            "output_playlist": bool(self.output_playlist),
               }
       
           @classmethod
      @@ -94,6 +98,8 @@ def from_dict(cls, data: Dict[str, Any] | None) -> "ScanSession":
                   scan_config=ScanConfig.from_dict(data.get("scan_config")),
                   scan_state=scan_state,
                   exported_report_version=int(data.get("exported_report_version", DEFAULT_REPORT_VERSION)),
      +            transfer_mode=str(data.get("transfer_mode", "copy") or "copy").lower(),
      +            output_playlist=bool(data.get("output_playlist", False)),
               )
       
       
      @@ -206,6 +212,9 @@ def __init__(
               self.preset_var = tk.StringVar(value=self.session.scan_config.preset_name or "")
               self.report_version_var = tk.StringVar(value=str(self.session.exported_report_version))
               self.scan_state_var = tk.StringVar(value=self._describe_state(self.session.scan_state))
      +        transfer_mode = (self.session.transfer_mode or "copy").lower()
      +        self.transfer_mode_var = tk.StringVar(value="copy" if transfer_mode == "copy" else "move")
      +        self.output_playlist_var = tk.BooleanVar(value=bool(getattr(self.session, "output_playlist", False)))
               self._on_close_callback = on_close
               self.library_var.trace_add("write", lambda *_: self._invalidate_plan("Library folder changed"))
               self.incoming_var.trace_add("write", lambda *_: self._invalidate_plan("Incoming folder changed"))
      @@ -419,14 +428,24 @@ def _build_plan_controls(self, parent: ttk.Frame) -> None:
               self.build_plan_btn.pack(side="left")
               self.preview_plan_btn = ttk.Button(btns, text="Preview", command=self._preview_plan)
               self.preview_plan_btn.pack(side="left", padx=(5, 0))
      +        self.transfer_mode_btn = ttk.Button(btns, command=self._toggle_transfer_mode)
      +        self.transfer_mode_btn.pack(side="left", padx=(5, 0))
               self.execute_plan_btn = ttk.Button(btns, text="Execute", command=self._execute_plan)
               self.execute_plan_btn.pack(side="left", padx=(5, 0))
      +        self.playlist_check = ttk.Checkbutton(
      +            btns,
      +            text="Output playlist",
      +            variable=self.output_playlist_var,
      +            command=self._on_output_playlist_toggle,
      +        )
      +        self.playlist_check.pack(side="left", padx=(10, 0))
               self.open_preview_btn = ttk.Button(
                   btns, text="Open Preview", command=self._open_preview_file, state="disabled"
               )
               self.open_preview_btn.pack(side="right")
               self.cancel_plan_btn = ttk.Button(btns, text="Cancel", command=self._cancel_plan_task, state="disabled")
               self.cancel_plan_btn.pack(side="right", padx=(5, 0))
      +        self._update_transfer_mode_label()
               self._refresh_plan_actions()
       
           def _build_incoming_tree(self, parent: ttk.Frame) -> None:
      @@ -513,6 +532,8 @@ def _persist_session(self) -> bool:
                   ),
                   scan_state=self.session.scan_state,
                   exported_report_version=report_ver,
      +            transfer_mode=self._current_transfer_mode(),
      +            output_playlist=bool(self.output_playlist_var.get()),
               )
               save_scan_session(self.session)
               self.scan_state_var.set(self._describe_state(self.session.scan_state))
      @@ -524,6 +545,8 @@ def _persist_session(self) -> bool:
                       "overrides": overrides,
                       "preset": self.preset_var.get().strip() or None,
                       "report_version": report_ver,
      +                "transfer_mode": self._current_transfer_mode(),
      +                "output_playlist": bool(self.output_playlist_var.get()),
                   },
               )
               return True
      @@ -541,6 +564,23 @@ def _invalidate_plan(self, reason: str | None = None) -> None:
                   self.plan_status_var.set(reason)
               self._refresh_plan_actions()
       
      +    def _current_transfer_mode(self) -> str:
      +        return "copy" if self.transfer_mode_var.get().lower() == "copy" else "move"
      +
      +    def _update_transfer_mode_label(self) -> None:
      +        label = "Copy Originals" if self._current_transfer_mode() == "copy" else "Move Originals"
      +        self.transfer_mode_btn.config(text=label)
      +
      +    def _toggle_transfer_mode(self) -> None:
      +        new_mode = "move" if self._current_transfer_mode() == "copy" else "copy"
      +        self.transfer_mode_var.set(new_mode)
      +        self._update_transfer_mode_label()
      +        self._invalidate_plan("Transfer mode changed")
      +        self._persist_session()
      +
      +    def _on_output_playlist_toggle(self) -> None:
      +        self._persist_session()
      +
           def _refresh_plan_actions(self) -> None:
               """Enable/disable plan controls based on current state."""
               running = self.plan_running
      @@ -554,9 +594,13 @@ def _refresh_plan_actions(self) -> None:
                   and sources_match
               )
       
      +        self._update_transfer_mode_label()
               for btn in (self.build_plan_btn, self.preview_plan_btn):
                   btn_state = "disabled" if running else "normal"
                   btn.config(state=btn_state)
      +        mode_state = "disabled" if running else "normal"
      +        self.transfer_mode_btn.config(state=mode_state)
      +        self.playlist_check.config(state="disabled" if running else "normal")
               self.cancel_plan_btn.config(state="normal" if running else "disabled")
               self.execute_plan_btn.config(state="normal" if execute_ready else "disabled")
               self.open_preview_btn.config(state="normal" if self.plan_preview_path else "disabled")
      @@ -587,10 +631,15 @@ def _start_plan_task(self, render_preview: bool) -> None:
               self.plan_progress_label.set("Starting…")
               self.plan_status_var.set("Building preview…" if render_preview else "Building plan…")
               self._refresh_plan_actions()
      -        thread = threading.Thread(target=self._plan_worker, args=(library_root, incoming_root, render_preview), daemon=True)
      +        transfer_mode = self._current_transfer_mode()
      +        thread = threading.Thread(
      +            target=self._plan_worker,
      +            args=(library_root, incoming_root, transfer_mode, render_preview),
      +            daemon=True,
      +        )
               thread.start()
       
      -    def _plan_worker(self, library_root: str, incoming_root: str, render_preview: bool) -> None:
      +    def _plan_worker(self, library_root: str, incoming_root: str, transfer_mode: str, render_preview: bool) -> None:
               docs_dir = os.path.join(library_root, "Docs")
               os.makedirs(docs_dir, exist_ok=True)
               output_html = os.path.join(docs_dir, "LibrarySyncPreview.html")
      @@ -610,6 +659,7 @@ def progress(current: int, total: int, path: str, phase: str) -> None:
                           log_callback=log,
                           progress_callback=progress,
                           cancel_event=self.plan_cancel,
      +                    transfer_mode=transfer_mode,
                       )
                       preview_path = output_html
                   else:
      @@ -619,6 +669,7 @@ def progress(current: int, total: int, path: str, phase: str) -> None:
                           log_callback=log,
                           progress_callback=progress,
                           cancel_event=self.plan_cancel,
      +                    transfer_mode=transfer_mode,
                       )
                       preview_path = None
               except IndexCancelled:
      @@ -651,7 +702,11 @@ def _on_plan_ready(
                   self._preview_version = self._plan_version
                   self.plan_preview_path = preview_path
                   self.plan_status_var.set(f"Preview ready: {preview_path}")
      -            self._log_event("plan_preview", "Preview ready", {"path": preview_path, "moves": len(plan.moves)})
      +            self._log_event(
      +                "plan_preview",
      +                "Preview ready",
      +                {"path": preview_path, "moves": len(plan.moves), "transfer_mode": plan.transfer_mode},
      +            )
                   if preview_path:
                       try:
                           webbrowser.open(preview_path)
      @@ -661,7 +716,11 @@ def _on_plan_ready(
                   self.plan_preview_path = None
                   self._preview_version = -1
                   self.plan_status_var.set("Plan built. Run Preview to inspect before execution.")
      -            self._log_event("plan_built", "Plan ready", {"moves": len(plan.moves)})
      +            self._log_event(
      +                "plan_built",
      +                "Plan ready",
      +                {"moves": len(plan.moves), "transfer_mode": plan.transfer_mode},
      +            )
       
               self.plan_progress_var.set(1)
               self.plan_progress_label.set("Complete")
      @@ -737,10 +796,15 @@ def _execute_plan(self) -> None:
               self.plan_progress_label.set("Executing…")
               self.plan_status_var.set("Executing plan…")
               self._refresh_plan_actions()
      -        thread = threading.Thread(target=self._execute_plan_worker, args=(self.active_plan,), daemon=True)
      +        create_playlist = bool(self.output_playlist_var.get())
      +        thread = threading.Thread(
      +            target=self._execute_plan_worker,
      +            args=(self.active_plan, create_playlist),
      +            daemon=True,
      +        )
               thread.start()
       
      -    def _execute_plan_worker(self, plan: library_sync.LibrarySyncPlan) -> None:
      +    def _execute_plan_worker(self, plan: library_sync.LibrarySyncPlan, create_playlist: bool) -> None:
               def log(msg: str) -> None:
                   self._log_event("execute_log", msg)
       
      @@ -753,6 +817,7 @@ def progress(current: int, total: int, path: str, phase: str) -> None:
                       log_callback=log,
                       progress_callback=progress,
                       cancel_event=self.plan_cancel,
      +                create_playlist=create_playlist,
                   )
               except Exception as exc:  # pragma: no cover - UI behavior
                   self.after(0, lambda e=exc: self._handle_plan_error(e))
      @@ -765,7 +830,10 @@ def _on_execute_complete(self, summary: Dict[str, object]) -> None:
               self.plan_cancel.clear()
               cancelled = bool(summary.get("cancelled"))
               moved = int(summary.get("moved", 0))
      +        copied = int(summary.get("copied", 0))
      +        transferred = int(summary.get("transferred", moved + copied))
               errors = summary.get("errors", []) or []
      +        playlist_path = summary.get("playlist_path")
       
               if cancelled:
                   status = "Execution cancelled."
      @@ -773,18 +841,53 @@ def _on_execute_complete(self, summary: Dict[str, object]) -> None:
                   self.plan_progress_label.set("Cancelled")
                   self._log_event("execute_cancelled", status)
               else:
      -            status = f"Execution complete. Moved {moved} files."
      +            parts = []
      +            if copied:
      +                parts.append(f"Copied {copied}")
      +            if moved:
      +                parts.append(f"Moved {moved}")
      +            if not parts:
      +                parts.append("No files transferred")
      +            status = "Execution complete. " + " and ".join(parts) + "."
                   self.plan_progress_var.set(1)
                   self.plan_progress_label.set("Complete")
      -            self._log_event("execute_complete", status, {"moved": moved})
      +            self._log_event(
      +                "execute_complete",
      +                status,
      +                {"moved": moved, "copied": copied, "transferred": transferred},
      +            )
       
               self.plan_status_var.set(status)
               if errors:
                   messagebox.showerror("Execution Errors", "\n".join(errors))
               elif not cancelled:
                   messagebox.showinfo("Execution Complete", status)
      +            if playlist_path:
      +                self._show_playlist_popup(str(playlist_path))
               self._refresh_plan_actions()
       
      +    def _show_playlist_popup(self, path: str) -> None:
      +        popup = tk.Toplevel(self)
      +        popup.title("Playlist Created")
      +        popup.transient(self.winfo_toplevel())
      +        ttk.Label(popup, text=f"Playlist saved:\n{path}", wraplength=420, justify="left").pack(
      +            padx=12, pady=(12, 8), anchor="w"
      +        )
      +        btns = ttk.Frame(popup)
      +        btns.pack(padx=12, pady=(0, 12), anchor="e")
      +
      +        def copy_path() -> None:
      +            try:
      +                popup.clipboard_clear()
      +                popup.clipboard_append(path)
      +            except Exception:
      +                pass
      +
      +        ttk.Button(btns, text="Copy Path", command=copy_path).pack(side="left")
      +        ttk.Button(btns, text="Close", command=popup.destroy).pack(side="left", padx=(6, 0))
      +        popup.grab_set()
      +        popup.lift()
      +
           def _describe_state(self, state: ScanState) -> str:
               return state.value.replace("_", " ").title()
       
      diff --git a/music_indexer_api.py b/music_indexer_api.py
      index ae2d511..7a6131c 100644
      --- a/music_indexer_api.py
      +++ b/music_indexer_api.py
      @@ -770,17 +770,19 @@ def render_dry_run_html_from_plan(
           """Render a dry-run preview from an already computed move plan."""
           coord = DryRunCoordinator()
       
      -    def _decision_annotation(decision: str):
      +    def _decision_annotation(decision: str, action: str | None = None):
      +        normalized_action = (action or "").lower()
      +        action_label = "(copy)" if normalized_action == "copy" else "(move)"
               mapping = {
                   "SKIP_DUPLICATE": ("(duplicate)", "decision-skip"),
                   "SKIP_KEEP_EXISTING": ("(kept existing)", "decision-skip"),
                   "REVIEW_REQUIRED": ("(review required)", "decision-review"),
                   "REPLACE": ("(replace)", "decision-replace"),
      -            "COPY": ("(move)", "decision-copy"),
      +            "COPY": (action_label, "decision-copy" if normalized_action == "copy" else "decision-move"),
               }
               return mapping.get(decision.upper())
       
      -    def _render_legend(decisions):
      +    def _render_legend(decisions, action_map=None):
               if not decisions:
                   return ""
               legend_lines = ["
      Legend:"] @@ -789,12 +791,15 @@ def _render_legend(decisions): "SKIP_KEEP_EXISTING": "A different file is already present; keeping existing.", "REVIEW_REQUIRED": "Needs manual approval before moving.", "REPLACE": "Existing file will be replaced.", - "COPY": "Incoming file will be moved.", + "COPY": "Incoming file will be transferred.", } for key in ["SKIP_DUPLICATE", "SKIP_KEEP_EXISTING", "REVIEW_REQUIRED", "REPLACE", "COPY"]: if key not in decisions: continue - annotation = _decision_annotation(key) + action = None + if action_map and key in action_map and action_map[key]: + action = sorted(action_map[key])[0] + annotation = _decision_annotation(key, action or "move") if not annotation: continue label, css = annotation @@ -810,16 +815,20 @@ def build_exact_metadata_section() -> str: lines = [f"

      {heading_text}

      "] decision_map = {} decision_keys = set() + decision_actions = {} if plan_items: for item in plan_items: dst = item.get("destination") decision = (item.get("decision") or "").upper() - annotation = _decision_annotation(decision) + annotation = _decision_annotation(decision, item.get("action")) if dst and annotation: decision_map[dst] = annotation decision_keys.add(decision) + action_val = (item.get("action") or "").lower() + if action_val: + decision_actions.setdefault(decision, set()).add(action_val) - legend_html = _render_legend(decision_keys) + legend_html = _render_legend(decision_keys, decision_actions if decision_actions else None) if legend_html: lines.append(legend_html) lines.append("
      ")
      @@ -865,7 +874,7 @@ def build_exact_metadata_section() -> str:
               out.write(
                   "\n\n\n  \n  "
                   f"{title_prefix} – {sanitize(os.path.basename(root_path))}\n  "
      -            "\n\n\n"
      +            "\n\n\n"
               )
               out.write(html_body)
               if html_body and not html_body.endswith("\n"):
      
      From 91ccbda4d1a9bd8c221fb08f982c0770a73ff26c Mon Sep 17 00:00:00 2001
      From: Greensand321 
      Date: Sat, 20 Dec 2025 17:46:07 -0500
      Subject: [PATCH 144/606] Update library sync and indexer documentation
      
      ---
       docs/library_sync_redesign.md   | 10 ++++++++++
       docs/project_documentation.html | 15 +++++++++++++++
       2 files changed, 25 insertions(+)
      
      diff --git a/docs/library_sync_redesign.md b/docs/library_sync_redesign.md
      index 4b1e387..7689c8b 100644
      --- a/docs/library_sync_redesign.md
      +++ b/docs/library_sync_redesign.md
      @@ -3,6 +3,8 @@
       ## Overview
       This redesign reframes the Library Sync tool as a review-first comparison interface between an existing music library and an incoming folder. The UI prioritizes clarity, safety, and trust by clearly distinguishing incoming content from existing content, surfacing duplicates and collisions, and enabling informed decisions through non-destructive review actions. All automated file operations and playlist updates are intentionally excluded from this tool.
       
      +**Backend alignment (current state):** The shipping backend in `library_sync.py` already follows this review-first contract. It performs independent scans of the *Existing Library* and *Incoming Folder*, reuses cached fingerprints where size/mtime match, computes new fingerprints in parallel when needed, and matches via indexed shortlists (fingerprint signature ➜ extension ➜ fallback pool). Matches are classified as *New*, *Collision*, *Exact Match*, or *Low Confidence* with a quality label (*Potential Upgrade* / *Keep Existing*) derived from format priority and bitrate. Execution helpers remain opt-in and write dry-run previews plus JSON/HTML audit logs; file operations are blocked unless a plan is explicitly executed.
      +
       ## User Flow (Start ➜ Finish)
       1. **Pick folders:** The user independently selects an *Existing Library* folder and an *Incoming Folder*.
       2. **Configure scan:** The user adjusts fingerprint distance thresholds (global or per-format) and optional quality tie-breaker settings, then starts scans for each folder independently.
      @@ -31,6 +33,8 @@ This redesign reframes the Library Sync tool as a review-first comparison interf
       - Shows detailed information for the selected item, including tags, duration, bitrate (when available), format-based quality score, fingerprint distance, threshold used, and the single best-matched counterpart (lowest distance). If multiple potential matches exist, only the best candidate is actionable; others are informational.
       - Optional shortcuts such as “open containing folder” and “play preview” are shown only when supported by the runtime environment.
       
      +**Implementation snapshot:** Each scan produces normalized `TrackRecord` entries (path, ext, bitrate, size, mtime, duration, fingerprint, tags). The inspector can display `MatchResult` payloads that already include the best match, candidate shortlist (with distances), threshold applied, near-miss margin, confidence score, and quality comparison labels. These payloads also flag when a result is partial due to cancellation, allowing the UI to show “Partial scan” states without blocking retries.
      +
       ### Status Chips
       - Each row may display one or more chips, including *New*, *Collision*, *Exact Match*, *Low Confidence*, *Potential Upgrade*, *Keep Existing*, or *Missing Metadata*. Chips are used consistently throughout the UI to avoid ambiguity.
       
      @@ -43,6 +47,10 @@ This redesign reframes the Library Sync tool as a review-first comparison interf
       - **Clear Flags:** Removes all review flags without affecting scan results.
       - No UI control, context menu, shortcut, or background task performs file moves, replacements, or playlist updates.
       
      +**Execution pipeline (opt-in):**
      +- **Plan generation:** `compute_library_sync_plan()` builds a deterministic move/route plan by reusing the indexer engine’s routing rules, remapping destinations under the selected library root, and attaching per-file decisions (*COPY*, *REPLACE*, or *SKIP* with reasons). Dry-run previews reuse the indexer HTML renderer for consistency.
      +- **Execution:** `execute_plan()` consumes the plan, writes JSON audit logs plus executed HTML reports, creates a “LibrarySync_Added_YYYY-mm-dd_hhmm.m3u8” playlist of transferred files (when enabled), and writes backups before replacements. Cancellations are honored between steps; skipped or review-required items are summarized in the report.
      +
       ### Progress and Status
       - Separate progress indicators are shown for library and incoming scans. Each scan can be cancelled independently. A log panel displays scan configuration, counts, skipped files, and warnings.
       
      @@ -69,6 +77,8 @@ This redesign reframes the Library Sync tool as a review-first comparison interf
       - Path and permission errors surface as inline banners and log entries without disabling retry.
       - Threshold inputs are validated; invalid values are rejected with inline feedback and reverted to the last known good configuration.
       
      +**Performance profiling hooks:** An optional `PerformanceProfile` collector records cache hits, fingerprint computations, shortlist sizes, and scan/match durations. UI surfaces can render these metrics for troubleshooting slow comparisons without affecting the main workflow.
      +
       ## Export Plan Schema (Minimum)
       - Exported reports must include incoming path, existing path (nullable), match status, fingerprint distance, threshold used, quality scores, user flags, notes, timestamp, and a scan configuration hash.
       
      diff --git a/docs/project_documentation.html b/docs/project_documentation.html
      index ba7bec0..b8f7a8a 100644
      --- a/docs/project_documentation.html
      +++ b/docs/project_documentation.html
      @@ -35,6 +35,8 @@ 

      2.1 Music Indexer (music_indexer_api.py)

    • Scans folders recursively for audio files.
    • Reads metadata: artist, title, album, year, track number, embedded cover art.
    • Deduplicates tracks by fingerprint, preferring FLAC > M4A > MP3.
    • +
    • Separates workflow into deterministic phases: vault pre-scan (artist counts), fingerprint-based dedupe, metadata aggregation, destination routing, and preview generation.
    • +
    • Surfaces every routing decision in decision_log and a tag index to drive consistent HTML previews and downstream plan rendering.
    • Phase 0: Pre-scan the vault under MUSIC_ROOT to build a global artist-count map via build_primary_counts().
    • Implements grouping rules:
        @@ -120,6 +122,19 @@

        2.1 Music Indexer (music_indexer_api.py)


        +

        2.5 Library Sync Review Tool (library_sync.py)

        +

        Comparison-first workflow that aligns to the redesign in docs/library_sync_redesign.md:

        +
          +
        • Runs independent scans for Library and Incoming folders, reusing cached fingerprints when size/mtime match and computing new prints in parallel when needed.
        • +
        • Builds normalized TrackRecord entries (path, ext, bitrate, size, mtime, duration, fingerprint, tags) and matches via indexed shortlists: fingerprint signature → extension cohort → fallback pool.
        • +
        • Classifies each incoming track as New, Collision, Exact Match, or Low Confidence, and attaches quality labels (Potential Upgrade / Keep Existing) using format priority and bitrate.
        • +
        • Exposes full MatchResult payloads with shortlist distances, thresholds used, near-miss margins, and confidence scores so the UI can render detailed inspectors without extra computation.
        • +
        • Optional profiling (PerformanceProfile) tracks cache hits, fingerprints computed, shortlist sizes, and scan/match timings for troubleshooting.
        • +
        • Opt-in execution path: compute_library_sync_plan() remaps indexer-derived destinations under the chosen library root and emits deterministic plan items (COPY / REPLACE / SKIP with reasons). execute_plan() writes JSON audits + executed HTML reports, creates a playlist of transferred files, and backs up any replaced destination before writing.
        • +
        + +
        +

        2.2 Fingerprint Generator (fingerprint_generator.py)

        Walks a folder tree and stores AcoustID fingerprints for each audio file in a SQLite database.

        From c949406e7a09e18b3104bb510bb54113124eaea9 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 21 Dec 2025 12:28:28 -0500 Subject: [PATCH 145/606] Replace Quality Checker with Duplicate Finder stub --- README.md | 9 +- main_gui.py | 384 +++++++++++----------------------------------------- 2 files changed, 85 insertions(+), 308 deletions(-) diff --git a/README.md b/README.md index 7b0358e..3a20d53 100644 --- a/README.md +++ b/README.md @@ -25,10 +25,9 @@ pip install -r requirements.txt The indexer will exit with an error if the real `mutagen` package is missing, so ensure all dependencies are installed before running. -The **Quality Checker** now imports the indexer's `fingerprint_generator` -module. Make sure the Music Indexer package itself is installed (e.g. -`pip install .`) so this dependency is available when running the Quality -Checker. +The **Duplicate Finder** tab now opens the redesigned shell for spotting +duplicates. Keep the Music Indexer package installed (e.g. `pip install .`) so +the duplicate detection backend remains available as the new UI rolls out. ### Optional: Essentia audio engine @@ -67,7 +66,7 @@ python main_gui.py 9. **Library Duplicate Scan** finds duplicate tracks after you drop new songs directly into your library 10. **Cross-Album Scan** optionally finds duplicates appearing on multiple albums 11. Use the **Theme** dropdown and **Help** tab for assistance -12. Adjust the **Distance Threshold** in the Quality Checker tab to fine‑tune duplicate detection and view detailed fingerprint distance logs +12. Launch the **Duplicate Finder** tab to open the updated shell for spotting duplicates ### Playlist generator feedback diff --git a/main_gui.py b/main_gui.py index 9905610..0d27863 100644 --- a/main_gui.py +++ b/main_gui.py @@ -41,11 +41,7 @@ find_duplicates as api_find_duplicates, get_tags, ) -import simple_duplicate_finder as sdf_mod -import fingerprint_cache -import chromaprint_utils from controllers.library_index_controller import generate_index -from controllers.scan_progress_controller import ScanProgressController from gui.audio_preview import PlaybackError, VlcPreviewPlayer from io import BytesIO from PIL import Image, ImageTk @@ -79,7 +75,7 @@ from plugins.acoustid_plugin import AcoustIDService, MusicBrainzService from controllers.cluster_controller import cluster_library from controllers.cluster_view_controller import ClusterComputationManager -from config import load_config, save_config, DEFAULT_FP_THRESHOLDS +from config import load_config, save_config import playlist_engine from playlist_engine import bucket_by_tempo_energy, autodj_playlist from controllers.cluster_controller import gather_tracks @@ -1447,62 +1443,46 @@ def update_progress(self, value): self.update_idletasks() -class ScanProgressWindow(tk.Toplevel): - """Non-modal window showing scan progress and logs.""" +class DuplicateFinderShell(tk.Toplevel): + """Placeholder shell for the updated Duplicate Finder UI.""" - def __init__(self, parent: tk.Widget, cancel_event: threading.Event): + def __init__(self, parent: tk.Widget, library_path: str): super().__init__(parent) - self.title("Scanning…") - self.cancel_event = cancel_event - self.resizable(True, True) + self.title("Duplicate Finder") self.transient(parent) + self.resizable(True, False) - self.progress = ttk.Progressbar(self, mode="indeterminate") - self.progress.pack(fill="x", padx=10, pady=(10, 0)) - self.progress.start() - - self.log_widget = ScrolledText(self, width=60, height=15, state="disabled") - self.log_widget.pack(padx=10, pady=10, fill="both", expand=True) - - btn = ttk.Button(self, text="Cancel", command=self._on_cancel) - btn.pack(pady=(0, 10)) - self.protocol("WM_DELETE_WINDOW", self._on_cancel) - - def _on_cancel(self) -> None: - if messagebox.askyesno("Cancel Scan", "Stop scanning?"): - self.cancel_event.set() - - # Public method used by background threads - def update_progress(self, kind: str, current: int, total: int, msg: str) -> None: - self.after(0, lambda: self._do_update(kind, current, total, msg)) - - # Actual UI updates executed on main thread - def _do_update(self, kind: str, current: int, total: int, msg: str) -> None: - if kind == "walk": - if self.progress["mode"] != "indeterminate": - self.progress.config(mode="indeterminate") - self.progress.start() - if msg: - self._append(f"Scanning: {msg}") - elif kind == "fp_start": - if self.progress["mode"] != "determinate": - self.progress.stop() - self.progress.config(mode="determinate", maximum=total, value=current - 1) - self._append(f"Fingerprinting file {current} of {total}\n{msg}") - elif kind == "fp_end": - if self.progress["mode"] == "determinate": - self.progress["value"] = current - elif kind == "log": - self._append(msg) - elif kind == "complete": - self.progress.stop() - self._append(f"Completed: {current} duplicate pairs found") - - def _append(self, text: str) -> None: - self.log_widget.configure(state="normal") - self.log_widget.insert("end", text + "\n") - self.log_widget.see("end") - self.log_widget.configure(state="disabled") + ttk.Label( + self, + text="Duplicate Finder", + font=("TkDefaultFont", 11, "bold"), + ).pack(anchor="w", padx=10, pady=(10, 4)) + ttk.Label( + self, + text=( + "The Duplicate Finder is being redesigned. " + "This stub window replaces the previous tool while the new experience is built." + ), + wraplength=460, + justify="left", + ).pack(anchor="w", padx=10, pady=(0, 8)) + + ttk.Label( + self, + text=f"Library: {library_path or 'No library selected'}", + wraplength=460, + justify="left", + ).pack(anchor="w", padx=10, pady=(0, 8)) + + ttk.Label( + self, + text="No duplicate scans are performed from this stub.", + foreground="#666", + wraplength=460, + justify="left", + ).pack(anchor="w", padx=10) + + ttk.Button(self, text="Close", command=self.destroy).pack(pady=(0, 10)) class SoundVaultImporterApp(tk.Tk): def __init__(self): @@ -1521,7 +1501,7 @@ def __init__(self): self.library_path = "" self.library_name_var = tk.StringVar(value="No library selected") self.library_path_var = tk.StringVar(value="") - # Folder to run Quality Checker - now always uses library_path + # Folder to run Duplicate Finder - now always uses library_path self.dup_folder_var = tk.StringVar(value="") # retained for compatibility self.library_stats_var = tk.StringVar(value="") self.show_all = False @@ -1561,16 +1541,10 @@ def __init__(self): self.tagfix_debug_var = tk.BooleanVar(value=False) self.tagfix_api_status = tk.StringVar(value="") - self.dup_debug_var = tk.BooleanVar(value=False) - - # Distance threshold for duplicate scanning - dup_thr = cfg.get("duplicate_threshold", 0.03) - self.fp_threshold_var = tk.DoubleVar(value=dup_thr) - dup_pref = cfg.get("duplicate_prefix_len", sdf_mod.FP_PREFIX_LEN) - self.fp_prefix_var = tk.IntVar(value=dup_pref) + # Duplicate Finder state + self.duplicate_finder_window: DuplicateFinderShell | None = None - # Quality Checker state - self._dup_logging = False + # Shared preview/playback state self._preview_thread = None self.player_status_var = tk.StringVar( value="Select a library to load tracks." @@ -1586,7 +1560,6 @@ def __init__(self): self.player_status_var.set(f"Preview disabled: {reason}") else: logging.info("VLC preview backend initialized") - self._quality_play_buttons: list[ttk.Button] = [] self._preview_in_progress = False self._player_busy_item: str | None = None self._ignore_next_preview_finish = False @@ -1973,50 +1946,33 @@ def build_ui(self): apply_frame, text="Apply Selected", command=self._apply_selected_tags ).pack(side="right") - # ─── Quality Checker Tab ───────────────────────────────────────── + # ─── Duplicate Finder Tab ───────────────────────────────────────── self.dup_tab = ttk.Frame(self.notebook) - self.notebook.add(self.dup_tab, text="Quality Checker") + self.notebook.add(self.dup_tab, text="Duplicate Finder") - df_controls = ttk.Frame(self.dup_tab) - df_controls.pack(fill="x", padx=10, pady=(10, 5)) - ttk.Label(df_controls, textvariable=self.library_path_var).pack(side="left") + df_container = ttk.LabelFrame(self.dup_tab, text="Duplicate Finder") + df_container.pack(fill="both", expand=True, padx=10, pady=10) + ttk.Label( + df_container, + textvariable=self.library_path_var, + justify="left", + ).pack(anchor="w", padx=10, pady=(8, 4)) + ttk.Label( + df_container, + text=( + "The Duplicate Finder is being refreshed. Launch the new shell to try " + "the updated experience when it becomes available." + ), + wraplength=520, + justify="left", + ).pack(anchor="w", padx=10, pady=(0, 6)) self.scan_btn = ttk.Button( - df_controls, - text="Scan", + df_container, + text="Open Duplicate Finder", command=self.scan_duplicates, state="disabled", ) - self.scan_btn.pack(side="left", padx=(5, 0)) - ttk.Label(df_controls, text="Distance Threshold:").pack(side="left", padx=(10, 0)) - thr_entry = ttk.Entry(df_controls, textvariable=self.fp_threshold_var, width=5) - thr_entry.pack(side="left") - ttk.Label(df_controls, text="Prefix Length:").pack(side="left", padx=(10, 0)) - pref_entry = ttk.Entry(df_controls, textvariable=self.fp_prefix_var, width=4) - pref_entry.pack(side="left") - self.fp_threshold_var.trace_add("write", lambda *a: self._validate_threshold()) - self.fp_prefix_var.trace_add("write", lambda *a: self._validate_threshold()) - ttk.Checkbutton( - df_controls, - text="Verbose Debug", - variable=self.dup_debug_var, - ).pack(side="left", padx=(5, 0)) - - self.qc_canvas = tk.Canvas(self.dup_tab) - self.qc_scroll = ttk.Scrollbar( - self.dup_tab, orient="vertical", command=self.qc_canvas.yview - ) - self.qc_canvas.configure(yscrollcommand=self.qc_scroll.set) - self.qc_canvas.pack(side="left", fill="both", expand=True, padx=(10, 0), pady=(0, 10)) - self.qc_scroll.pack(side="right", fill="y", pady=(0, 10)) - self.qc_inner = ttk.Frame(self.qc_canvas) - self.qc_canvas.create_window((0, 0), window=self.qc_inner, anchor="nw") - self.qc_inner.bind( - "", - lambda e: self.qc_canvas.configure(scrollregion=self.qc_canvas.bbox("all")), - ) - if self.library_path_var.get(): - self.scan_btn.config(state="normal") - self._validate_threshold() + self.scan_btn.pack(anchor="w", padx=10, pady=(4, 10)) self._check_tagfix_api_status() # ─── Player Tab ──────────────────────────────────────────────── @@ -2232,7 +2188,6 @@ def select_library(self): self.active_plugin = None if hasattr(self, "scan_btn"): self.scan_btn.config(state="normal") - self._validate_threshold() self.update_library_info() if hasattr(self, "player_reload_btn"): self.player_reload_btn.config(state="normal") @@ -2245,6 +2200,8 @@ def update_library_info(self): self.library_stats_var.set("") if hasattr(self, "player_reload_btn"): self.player_reload_btn.config(state="disabled") + if hasattr(self, "scan_btn"): + self.scan_btn.config(state="disabled") self.player_status_var.set("Select a library to load tracks.") return num = count_audio_files(self.library_path) @@ -2319,70 +2276,27 @@ def validate_library(self): self._log(f"✘ Invalid SoundVault: {path}\n" + "\n".join(errors)) self.update_library_info() - # ── Quality Checker Actions ──────────────────────────────────────── def scan_duplicates(self): - folder = self.library_path_var.get() - if not folder: - messagebox.showwarning("No Folder", "Please select a library first.") + library = self.require_library() + if not library: return - debug_enabled = self.dup_debug_var.get() - sdf_mod.verbose = debug_enabled - fingerprint_cache.verbose = debug_enabled - chromaprint_utils.verbose = debug_enabled - if debug_enabled: - self._open_dup_debug_window() - - self.clear_quality_view() - self.scan_btn.config(state="disabled") - try: - thr = float(self.fp_threshold_var.get()) - except Exception: - thr = load_config().get("duplicate_threshold", 0.03) - self._log("! Invalid threshold; using saved value") - else: - if not (0.0 < thr <= 1.0): - thr = load_config().get("duplicate_threshold", 0.03) - self._log("! Threshold out of range; using saved value") - cfg = load_config() - cfg["duplicate_threshold"] = thr - try: - pref_val = int(self.fp_prefix_var.get()) - except Exception: - pref_val = sdf_mod.FP_PREFIX_LEN - self._log("! Invalid prefix length; using saved value") - cfg["duplicate_prefix_len"] = pref_val - save_config(cfg) - controller = ScanProgressController() - prog_win = ScanProgressWindow(self, controller.cancel_event) - controller.set_callback(prog_win.update_progress) - - def task(): - self._dup_logging = True - - def cb(msg): - self.after(0, lambda m=msg: self._log(m)) - controller.update("log", 0, 0, msg) - - try: - dups, missing = sdf_mod.find_duplicates( - folder, - threshold=thr, - prefix_len=pref_val, - log_callback=cb, - progress_callback=controller.update, - cancel_event=controller.cancel_event, - ) - self.after(0, lambda: self._log(f"Found {len(dups)} duplicate pairs")) - self.after(0, lambda: self.populate_quality_table(dups)) - if missing: - msg = f"{missing} files could not be fingerprinted." - self.after(0, lambda m=msg: self._log(m)) - finally: - self._dup_logging = False - self.after(0, lambda: (self.scan_btn.config(state="normal"), prog_win.destroy())) + existing = getattr(self, "duplicate_finder_window", None) + if existing and existing.winfo_exists(): + existing.lift() + existing.focus_set() + return - threading.Thread(target=task, daemon=True).start() + win = DuplicateFinderShell(self, library_path=library) + win.bind( + "", + lambda e: ( + setattr(self, "duplicate_finder_window", None) + if e.widget is win + else None + ), + ) + self.duplicate_finder_window = win def run_indexer(self): path = self.require_library() @@ -3437,19 +3351,6 @@ def initialize_genre_normalizer(self, mapping: dict, progress_callback=None): return changed, total - def _open_dup_debug_window(self) -> None: - if ( - getattr(self, "dup_debug_win", None) - and self.dup_debug_win.winfo_exists() - ): - return - win = tk.Toplevel(self) - win.title("Duplicate Finder Debug") - text = ScrolledText(win, height=20, wrap="word") - text.pack(fill="both", expand=True) - self.dup_debug_win = win - self.dup_text = text - def apply_mapping(self): """Persist mapping JSON from text box and apply normalization.""" if not hasattr(self, "text_map"): @@ -3552,29 +3453,6 @@ def _open_library_sync_tool(self) -> None: "Library Sync", "The Library Sync tab is not available." ) - # ── Quality Checker Helpers ───────────────────────────────────────── - def clear_quality_view(self) -> None: - for w in self.qc_inner.winfo_children(): - w.destroy() - - def _validate_threshold(self) -> None: - """Enable Scan button only when threshold is valid.""" - try: - val = float(self.fp_threshold_var.get()) - valid = 0.0 < val <= 1.0 - except Exception: - valid = False - try: - pref = int(self.fp_prefix_var.get()) - valid_pref = pref >= 0 - except Exception: - valid_pref = False - folder_selected = self.library_path_var.get() - if valid and valid_pref and folder_selected: - self.scan_btn.config(state="normal") - else: - self.scan_btn.config(state="disabled") - def _play_preview( self, path: str, @@ -3629,7 +3507,6 @@ def _preview_finished_ui(self, error: str | None = None): self._preview_in_progress = False self._restore_player_play_icons() - self._set_preview_controls_state(enabled=self.preview_backend_available) if hasattr(self, "player_status_var"): if error: self.player_status_var.set("Playback failed. Check logs.") @@ -3637,14 +3514,6 @@ def _preview_finished_ui(self, error: str | None = None): else: self.player_status_var.set("Ready to play another track.") - def _set_preview_controls_state(self, enabled: bool) -> None: - state = "!disabled" if enabled else "disabled" - for btn in getattr(self, "_quality_play_buttons", []): - try: - btn.state([state]) - except Exception: - logging.exception("Failed to update preview button state") - def _set_player_play_state_busy(self, item_id: str | None) -> None: if not hasattr(self, "player_tree"): return @@ -3919,92 +3788,6 @@ def _open_folder(self, path: str) -> None: except Exception as exc: messagebox.showerror("Open Folder", f"Could not open {path}: {exc}") - def populate_quality_table(self, matches): - self.clear_quality_view() - self._quality_play_buttons = [] - if not matches: - ttk.Label(self.qc_inner, text="No duplicates detected in this library").pack(pady=20) - return - - for keep_path, dup_path in matches: - row = ttk.Frame(self.qc_inner) - row.pack(fill="x", padx=10, pady=2) - - btn_state = "normal" - if not self.preview_backend_available or self._preview_in_progress: - btn_state = "disabled" - play_btn = ttk.Button( - row, - text="▶", - width=1, - state=btn_state, - command=lambda p=dup_path: self._play_preview(p), - ) - play_btn.pack(side="left") - self._quality_play_buttons.append(play_btn) - - thumb = self._load_thumbnail(dup_path, size=40) - img_label = ttk.Label(row, image=thumb) - img_label.image = thumb - img_label.pack(side="left", padx=(5, 10)) - - tags = get_tags(dup_path) - title = tags.get("title") or os.path.basename(dup_path) - artist = tags.get("artist") or "Unknown" - year = tags.get("year") or "?" - ext = os.path.splitext(dup_path)[1].lower() - size = os.path.getsize(dup_path) / 1024 / 1024 - - info = ttk.Frame(row) - info.pack(side="left", fill="x", expand=True) - ttk.Label(info, text=title, font=("TkDefaultFont", 9, "bold"), anchor="w").pack(anchor="w") - ttk.Label(info, text=f"{artist} • {year}", font=("TkDefaultFont", 8), anchor="w").pack(anchor="w") - ttk.Label(info, text=f"{ext[1:].upper()} {size:.1f} MB", font=("TkDefaultFont", 8), anchor="w").pack(anchor="w") - - del_btn = ttk.Button(row, text="Delete", command=lambda p=dup_path, f=row: self._prompt_delete(p, f)) - del_btn.pack(side="right") - - self.qc_canvas.update_idletasks() - - def _prompt_delete(self, path: str, row: tk.Widget) -> None: - if getattr(self, "_skip_delete_confirm", False): - proceed = True - else: - proceed, skip = self._confirm_delete(path) - if skip: - self._skip_delete_confirm = True - if not proceed: - return - try: - os.remove(path) - row.destroy() - self._log(f"Deleted duplicate {path}") - except Exception as e: - messagebox.showerror("Delete Failed", str(e)) - - def _confirm_delete(self, path: str) -> tuple[bool, bool]: - top = tk.Toplevel(self) - top.title("Confirm Delete") - top.grab_set() - ttk.Label(top, text=f"Delete {os.path.basename(path)}?").pack(padx=10, pady=(10, 5)) - skip_var = tk.BooleanVar() - ttk.Checkbutton(top, text="Don't ask again this session", variable=skip_var).pack(padx=10, pady=(0, 5), anchor="w") - result = {"ok": False} - - def do_ok() -> None: - result["ok"] = True - top.destroy() - - def do_cancel() -> None: - top.destroy() - - btn_frame = ttk.Frame(top) - btn_frame.pack(pady=(0, 10)) - ttk.Button(btn_frame, text="Delete", command=do_ok).pack(side="left", padx=5) - ttk.Button(btn_frame, text="Cancel", command=do_cancel).pack(side="left", padx=5) - top.wait_window() - return result["ok"], skip_var.get() - def _create_player_library_table(self, parent: tk.Widget) -> ttk.Frame: table = ttk.Frame(parent) p_vsb = ttk.Scrollbar(table, orient="vertical") @@ -4497,11 +4280,6 @@ def _log(self, msg): self.output.insert("end", line + "\n") self.output.see("end") self.output.configure(state="disabled") - if getattr(self, "_dup_logging", False) and hasattr(self, "dup_text"): - self.dup_text.configure(state="normal") - self.dup_text.insert("end", line + "\n") - self.dup_text.see("end") - self.dup_text.configure(state="disabled") if __name__ == "__main__": From bb014474b9b92deddf2ab42b5ecdd429c0e1ea98 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 21 Dec 2025 20:06:18 -0500 Subject: [PATCH 146/606] Add Duplicate Finder shell UI scaffold --- main_gui.py | 234 +++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 206 insertions(+), 28 deletions(-) diff --git a/main_gui.py b/main_gui.py index 0d27863..a0b276d 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1444,45 +1444,223 @@ def update_progress(self, value): class DuplicateFinderShell(tk.Toplevel): - """Placeholder shell for the updated Duplicate Finder UI.""" + """GUI-only shell for the refreshed Duplicate Finder workflow.""" def __init__(self, parent: tk.Widget, library_path: str): super().__init__(parent) self.title("Duplicate Finder") self.transient(parent) - self.resizable(True, False) + self.resizable(True, True) + + self.status_var = tk.StringVar(value="Idle") + self.progress_var = tk.DoubleVar(value=0) + self.library_path_var = tk.StringVar(value=library_path or "") + self.playlist_path_var = tk.StringVar( + value=self._default_playlist_folder(library_path) + ) + self.update_playlists_var = tk.BooleanVar(value=False) + self.quarantine_var = tk.BooleanVar(value=True) + + container = ttk.Frame(self) + container.pack(fill="both", expand=True, padx=10, pady=10) ttk.Label( - self, + container, text="Duplicate Finder", - font=("TkDefaultFont", 11, "bold"), - ).pack(anchor="w", padx=10, pady=(10, 4)) + font=("TkDefaultFont", 12, "bold"), + ).pack(anchor="w") ttk.Label( - self, - text=( - "The Duplicate Finder is being redesigned. " - "This stub window replaces the previous tool while the new experience is built." - ), - wraplength=460, - justify="left", - ).pack(anchor="w", padx=10, pady=(0, 8)) + container, + text="UI shell only – actions are stubbed and do not change your library.", + foreground="#555", + wraplength=520, + ).pack(anchor="w", pady=(0, 8)) + + # Library selection + lib_frame = ttk.LabelFrame(container, text="Library Selection") + lib_frame.pack(fill="x", pady=(0, 10)) + + lib_row = ttk.Frame(lib_frame) + lib_row.pack(fill="x", padx=8, pady=6) + ttk.Label(lib_row, text="Library Root").pack(side="left") + lib_entry = ttk.Entry(lib_row, textvariable=self.library_path_var, width=60) + lib_entry.pack(side="left", padx=6, expand=True, fill="x") + ttk.Button(lib_row, text="Browse…", command=self._browse_library).pack( + side="left" + ) - ttk.Label( - self, - text=f"Library: {library_path or 'No library selected'}", - wraplength=460, - justify="left", - ).pack(anchor="w", padx=10, pady=(0, 8)) + playlist_row = ttk.Frame(lib_frame) + playlist_row.pack(fill="x", padx=8, pady=(0, 8)) + ttk.Label(playlist_row, text="Playlist Folder").pack(side="left") + playlist_entry = ttk.Entry( + playlist_row, textvariable=self.playlist_path_var, width=60 + ) + playlist_entry.pack(side="left", padx=6, expand=True, fill="x") + ttk.Button(playlist_row, text="Browse…", command=self._browse_playlist).pack( + side="left" + ) + + # Controls + controls = ttk.Frame(container) + controls.pack(fill="x", pady=(0, 10)) + ttk.Button(controls, text="Scan Library", command=self._handle_scan).pack( + side="left", padx=(0, 6) + ) + ttk.Button(controls, text="Preview", command=self._handle_preview).pack( + side="left", padx=(0, 6) + ) + ttk.Button(controls, text="Execute", command=self._handle_execute).pack( + side="left", padx=(0, 12) + ) + ttk.Checkbutton( + controls, + text="Update Playlists", + variable=self.update_playlists_var, + command=self._toggle_update_playlists, + ).pack(side="left", padx=(0, 10)) + ttk.Checkbutton( + controls, + text="Quarantine Duplicates", + variable=self.quarantine_var, + command=self._toggle_quarantine, + ).pack(side="left") + + # Progress + status + status_frame = ttk.Frame(container) + status_frame.pack(fill="x", pady=(0, 10)) + ttk.Progressbar( + status_frame, + maximum=100, + variable=self.progress_var, + ).pack(fill="x", padx=(0, 10), side="left", expand=True) + ttk.Label(status_frame, textvariable=self.status_var, width=18).pack( + side="left" + ) + # Log box + log_box = ttk.LabelFrame(container, text="Log") + log_box.pack(fill="both", expand=True, pady=(0, 10)) + self.log_text = ScrolledText(log_box, height=6, state="disabled") + self.log_text.pack(fill="both", expand=True, padx=6, pady=6) + + # Results area + results = ttk.LabelFrame(container, text="Results") + results.pack(fill="both", expand=True) + results.columnconfigure(0, weight=1) + results.columnconfigure(1, weight=1) + + groups_frame = ttk.LabelFrame(results, text="Duplicate Groups") + groups_frame.grid(row=0, column=0, sticky="nsew", padx=(6, 3), pady=6) + cols = ("group", "title", "count", "status") + self.groups_tree = ttk.Treeview( + groups_frame, + columns=cols, + show="headings", + height=8, + ) + for cid, heading in zip( + cols, ("Group ID", "Track Title", "Count", "Status") + ): + self.groups_tree.heading(cid, text=heading) + width = 90 if cid in ("group", "count") else 160 + self.groups_tree.column(cid, width=width, anchor="w") + self.groups_tree.pack(fill="both", expand=True, padx=6, pady=6) + + inspector = ttk.LabelFrame(results, text="Group Details") + inspector.grid(row=0, column=1, sticky="nsew", padx=(3, 6), pady=6) ttk.Label( - self, - text="No duplicate scans are performed from this stub.", - foreground="#666", - wraplength=460, - justify="left", - ).pack(anchor="w", padx=10) + inspector, + text="Select a group to view details", + justify="center", + foreground="#555", + ).pack(fill="both", expand=True, padx=12, pady=12) + + self._log_action("Duplicate Finder shell initialized") + + def _default_playlist_folder(self, library_path: str) -> str: + if library_path: + candidate = os.path.join(library_path, "Playlists") + if os.path.isdir(candidate): + return candidate + return library_path + return "" + + def _log_action(self, message: str) -> None: + timestamp = datetime.now().strftime("%H:%M:%S") + line = f"{timestamp} {message}" + self.log_text.configure(state="normal") + self.log_text.insert("end", line + "\n") + self.log_text.see("end") + self.log_text.configure(state="disabled") + + def _set_status(self, status: str, progress: float | None = None) -> None: + self.status_var.set(status) + if progress is not None: + self.progress_var.set(progress) + + def _validate_library_root(self) -> str | None: + path = self.library_path_var.get().strip() + if not path: + messagebox.showwarning( + "Library Required", "Please select a library root before continuing." + ) + self._log_action("Library validation failed: no library selected") + self._set_status("Idle", progress=0) + return None + if not os.path.isdir(path): + messagebox.showwarning( + "Library Missing", f"The selected library path does not exist:\n{path}" + ) + self._log_action(f"Library validation failed: path not found ({path})") + self._set_status("Idle", progress=0) + return None + return path + + def _browse_library(self) -> None: + chosen = filedialog.askdirectory(title="Select Library Root") + if not chosen: + return + self.library_path_var.set(chosen) + self.playlist_path_var.set(self._default_playlist_folder(chosen)) + self._log_action(f"Library path updated to {chosen}") + + def _browse_playlist(self) -> None: + chosen = filedialog.askdirectory(title="Select Playlist Folder") + if not chosen: + return + self.playlist_path_var.set(chosen) + self._log_action(f"Playlist folder updated to {chosen}") + + def _handle_scan(self) -> None: + path = self._validate_library_root() + if not path: + return + self._log_action("Scan clicked") + self._set_status("Scanning", progress=25) + self._set_status("Ready", progress=100) + self._log_action("Scan stub completed (no files touched)") + + def _handle_preview(self) -> None: + path = self._validate_library_root() + if not path: + return + self._log_action("Preview clicked") + self._set_status("Preview generated", progress=100) + + def _handle_execute(self) -> None: + path = self._validate_library_root() + if not path: + return + self._log_action("Execute clicked") + self._set_status("Executed", progress=100) + + def _toggle_update_playlists(self) -> None: + state = "enabled" if self.update_playlists_var.get() else "disabled" + self._log_action(f"Update Playlists {state}") - ttk.Button(self, text="Close", command=self.destroy).pack(pady=(0, 10)) + def _toggle_quarantine(self) -> None: + state = "enabled" if self.quarantine_var.get() else "disabled" + self._log_action(f"Quarantine Duplicates {state}") class SoundVaultImporterApp(tk.Tk): def __init__(self): @@ -1960,8 +2138,8 @@ def build_ui(self): ttk.Label( df_container, text=( - "The Duplicate Finder is being refreshed. Launch the new shell to try " - "the updated experience when it becomes available." + "Preview the refreshed Duplicate Finder workflow. " + "This shell is UI-only for now and does not modify your library." ), wraplength=520, justify="left", From f710ff63613160e5c2ca1a8b2b84376964cb02e8 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 21 Dec 2025 20:29:35 -0500 Subject: [PATCH 147/606] Add duplicate consolidation planning module --- duplicate_consolidation.py | 387 +++++++++++++++++++++++++++++++++++++ 1 file changed, 387 insertions(+) create mode 100644 duplicate_consolidation.py diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py new file mode 100644 index 0000000..2691d87 --- /dev/null +++ b/duplicate_consolidation.py @@ -0,0 +1,387 @@ +"""Duplicate consolidation planning (no file mutations). + +This module performs a dry-run duplicate scan that groups musically identical +tracks using audio fingerprints, selects a deterministic "winner" based on +quality, and produces a consolidation plan describing artwork/metadata actions. +No files are moved or written—callers can safely surface the plan for review +before executing any changes. +""" +from __future__ import annotations + +import hashlib +import os +import threading +import time +from dataclasses import dataclass, field +from typing import Callable, Dict, Iterable, List, Mapping, MutableMapping, Optional, Sequence + +from near_duplicate_detector import fingerprint_distance + +LOSSLESS_EXTS = {".flac", ".wav", ".alac", ".ape", ".aiff", ".aif"} +DEFAULT_DISTANCE_THRESHOLD = 0.0 +DEFAULT_MAX_CANDIDATES = 5000 +DEFAULT_MAX_COMPARISONS = 50_000 +DEFAULT_TIMEOUT_SEC = 15.0 + + +def _now() -> float: + return time.perf_counter() + + +def _default_progress(_current: int, _total: int, _msg: str) -> None: + return None + + +@dataclass +class DuplicateTrack: + """Normalized representation of a track for consolidation planning.""" + + path: str + fingerprint: str | None + ext: str + bitrate: int = 0 + sample_rate: int = 0 + bit_depth: int = 0 + tags: Dict[str, object] = field(default_factory=dict) + + @property + def cover_hash(self) -> str | None: + value = self.tags.get("cover_hash") if isinstance(self.tags, Mapping) else None + if value: + return str(value) + art = self.tags.get("artwork_hash") if isinstance(self.tags, Mapping) else None + return str(art) if art else None + + @property + def is_lossless(self) -> bool: + return os.path.splitext(self.ext)[1].lower() in LOSSLESS_EXTS + + @property + def metadata_count(self) -> int: + if not isinstance(self.tags, Mapping): + return 0 + keys = [ + "artist", + "albumartist", + "title", + "album", + "track", + "tracknumber", + "disc", + "discnumber", + "date", + "year", + ] + return sum(1 for k in keys if self.tags.get(k)) + + +@dataclass +class ArtworkDirective: + """Instruction to copy artwork from a source to a target.""" + + source: str + target: str + reason: str + + def to_dict(self) -> Dict[str, str]: + return {"source": self.source, "target": self.target, "reason": self.reason} + + +@dataclass +class GroupPlan: + """Planned actions for a duplicate group.""" + + group_id: str + winner_path: str + losers: List[str] + planned_winner_tags: Dict[str, object] + artwork: List[ArtworkDirective] + loser_disposition: Dict[str, str] + playlist_rewrites: Dict[str, str] + review_flags: List[str] + context_summary: Dict[str, List[str]] + + def to_dict(self) -> Dict[str, object]: + return { + "group_id": self.group_id, + "winner_path": self.winner_path, + "losers": list(self.losers), + "planned_winner_tags": dict(self.planned_winner_tags), + "artwork": [a.to_dict() for a in self.artwork], + "loser_disposition": dict(self.loser_disposition), + "playlist_rewrites": dict(self.playlist_rewrites), + "review_flags": list(self.review_flags), + "context_summary": {k: list(v) for k, v in self.context_summary.items()}, + } + + +@dataclass +class ConsolidationPlan: + """Aggregate plan across all duplicate groups.""" + + groups: List[GroupPlan] = field(default_factory=list) + review_flags: List[str] = field(default_factory=list) + + def to_dict(self) -> Dict[str, object]: + return { + "groups": [g.to_dict() for g in self.groups], + "review_flags": list(self.review_flags), + } + + +def _normalize_track(raw: Mapping[str, object]) -> DuplicateTrack: + path = str(raw.get("path")) + ext = os.path.splitext(path)[1].lower() or str(raw.get("ext", "")).lower() + tags = raw.get("tags") if isinstance(raw.get("tags"), Mapping) else {} + return DuplicateTrack( + path=path, + fingerprint=raw.get("fingerprint"), + ext=ext, + bitrate=int(raw.get("bitrate") or 0), + sample_rate=int(raw.get("sample_rate") or raw.get("samplerate") or 0), + bit_depth=int(raw.get("bit_depth") or raw.get("bitdepth") or 0), + tags=dict(tags), + ) + + +def _classify_context(track: DuplicateTrack) -> str: + tags = track.tags or {} + album = str(tags.get("album") or "").strip() + album_type = str(tags.get("album_type") or tags.get("release_type") or "").lower() + track_no = tags.get("track") or tags.get("tracknumber") + disc_no = tags.get("disc") or tags.get("discnumber") + + if album_type == "single": + return "single" + if album_type in {"album", "lp"}: + return "album" + if album.lower().endswith(" - single") or "(single" in album.lower(): + return "single" + if album and (track_no or disc_no): + return "album" + if not album: + return "single" + if track.cover_hash: + return "album" + return "unknown" + + +def _quality_tuple(track: DuplicateTrack, context: str) -> tuple: + return ( + 1 if track.is_lossless else 0, + track.bitrate, + track.sample_rate, + track.bit_depth, + 1 if context == "single" else 0, + track.path.lower(), + ) + + +def _stable_group_id(paths: Sequence[str]) -> str: + canonical = "|".join(sorted(paths)) + return hashlib.sha1(canonical.encode("utf-8")).hexdigest()[:12] + + +def _select_metadata_source(candidates: Sequence[DuplicateTrack], contexts: Mapping[str, str]) -> DuplicateTrack | None: + album_candidates = [c for c in candidates if contexts.get(c.path) == "album"] + sorted_cands = sorted( + album_candidates or list(candidates), + key=lambda c: (c.metadata_count, c.bitrate, c.sample_rate, c.bit_depth, c.path.lower()), + reverse=True, + ) + return sorted_cands[0] if sorted_cands else None + + +def _merge_tags(existing: MutableMapping[str, object], source: Mapping[str, object]) -> Dict[str, object]: + merged = dict(existing) + for key, value in source.items(): + if key in merged and merged[key]: + continue + merged[key] = value + return merged + + +def _artwork_score(track: DuplicateTrack) -> tuple: + tags = track.tags or {} + size_hint = 0 + art_bytes = tags.get("artwork_bytes") if isinstance(tags, Mapping) else None + if isinstance(art_bytes, (bytes, bytearray)): + size_hint = len(art_bytes) + elif isinstance(art_bytes, int): + size_hint = art_bytes + return ( + 1 if track.cover_hash else 0, + size_hint, + track.bitrate, + track.sample_rate, + track.path.lower(), + ) + + +def _select_artwork_candidate(candidates: Sequence[DuplicateTrack]) -> tuple[DuplicateTrack | None, bool]: + ranked = sorted(candidates, key=_artwork_score, reverse=True) + if not ranked: + return None, False + if len(ranked) == 1: + return ranked[0], False + top_score = _artwork_score(ranked[0]) + ambiguous = sum(1 for c in ranked if _artwork_score(c) == top_score) > 1 + return ranked[0], ambiguous + + +def _cluster_duplicates( + tracks: Sequence[DuplicateTrack], + *, + threshold: float, + cancel_event: threading.Event, + max_comparisons: int, + timeout_sec: float, + progress_callback: Callable[[int, int, str], None], + start_time: float, + review_flags: List[str], +) -> List[List[DuplicateTrack]]: + buckets: Dict[str, List[DuplicateTrack]] = {} + for track in tracks: + if not track.fingerprint: + continue + key = str(track.fingerprint) if threshold <= 0 else str(track.fingerprint)[:20] + buckets.setdefault(key, []).append(track) + + clusters: List[List[DuplicateTrack]] = [] + comparisons = 0 + processed = 0 + bucket_items = sorted(buckets.items(), key=lambda x: x[0]) + for _, bucket_tracks in bucket_items: + if cancel_event.is_set() or (timeout_sec and (_now() - start_time) > timeout_sec): + review_flags.append("Consolidation planning cancelled or timed out during grouping.") + break + bucket_tracks = sorted(bucket_tracks, key=lambda t: t.path.lower()) + used: set[str] = set() + for idx, track in enumerate(bucket_tracks): + if ( + cancel_event.is_set() + or comparisons >= max_comparisons + or (timeout_sec and (_now() - start_time) > timeout_sec) + ): + break + if track.path in used: + continue + group = [track] + used.add(track.path) + for other in bucket_tracks[idx + 1 :]: + if cancel_event.is_set() or comparisons >= max_comparisons: + review_flags.append("Comparison budget reached; grouping may be incomplete.") + break + if timeout_sec and (_now() - start_time) > timeout_sec: + review_flags.append("Consolidation planning timed out while grouping.") + break + comparisons += 1 + dist = fingerprint_distance(track.fingerprint, other.fingerprint) + if dist <= threshold: + group.append(other) + used.add(other.path) + if len(group) > 1: + clusters.append(group) + processed += 1 + progress_callback(processed, len(bucket_tracks), track.path) + if cancel_event.is_set() or comparisons >= max_comparisons or (timeout_sec and (_now() - start_time) > timeout_sec): + break + return clusters + + +def build_consolidation_plan( + tracks: Iterable[Mapping[str, object]], + *, + distance_threshold: float = DEFAULT_DISTANCE_THRESHOLD, + max_candidates: int = DEFAULT_MAX_CANDIDATES, + max_comparisons: int = DEFAULT_MAX_COMPARISONS, + timeout_sec: float = DEFAULT_TIMEOUT_SEC, + cancel_event: Optional[threading.Event] = None, + progress_callback: Callable[[int, int, str], None] | None = None, +) -> ConsolidationPlan: + """Generate a deterministic consolidation plan without modifying files.""" + + cancel_event = cancel_event or threading.Event() + progress_callback = progress_callback or _default_progress + review_flags: List[str] = [] + start_time = _now() + + normalized: List[DuplicateTrack] = [] + for raw in tracks: + if cancel_event.is_set(): + review_flags.append("Cancelled before normalization.") + break + normalized.append(_normalize_track(raw)) + if len(normalized) >= max_candidates: + review_flags.append(f"Truncated candidate set to {max_candidates} items to protect runtime.") + break + + clusters = _cluster_duplicates( + normalized, + threshold=distance_threshold, + cancel_event=cancel_event, + max_comparisons=max_comparisons, + timeout_sec=timeout_sec, + progress_callback=progress_callback, + start_time=start_time, + review_flags=review_flags, + ) + + plans: List[GroupPlan] = [] + for cluster in sorted(clusters, key=lambda c: _stable_group_id([t.path for t in c])): + contexts = {t.path: _classify_context(t) for t in cluster} + quality_sorted = sorted(cluster, key=lambda t: _quality_tuple(t, contexts[t.path]), reverse=True) + winner = quality_sorted[0] + losers = [t.path for t in quality_sorted[1:]] + + metadata_source = _select_metadata_source(cluster, contexts) + planned_tags: Dict[str, object] = dict(winner.tags) + if metadata_source: + planned_tags = _merge_tags(planned_tags, metadata_source.tags) + if contexts.get(metadata_source.path) == "album": + planned_tags.setdefault("album_type", "album") + elif contexts.get(metadata_source.path) == "single": + planned_tags.setdefault("album_type", "single") + else: + review_flags.append(f"Missing metadata source for group containing {winner.path}.") + + single_candidates = [t for t in cluster if contexts.get(t.path) == "single"] + best_art, ambiguous_art = _select_artwork_candidate(single_candidates) + artwork_actions: List[ArtworkDirective] = [] + if best_art and (not winner.cover_hash or winner.cover_hash != best_art.cover_hash): + reason = "Copy single artwork to preserve release look" + artwork_actions.append(ArtworkDirective(source=best_art.path, target=winner.path, reason=reason)) + if ambiguous_art: + review_flags.append( + f"Artwork selection ambiguous for group {_stable_group_id([t.path for t in cluster])}." + ) + + dispositions = {loser: "quarantine" for loser in losers} + playlist_map = {loser: winner.path for loser in losers} + + group_id = _stable_group_id([t.path for t in cluster]) + group_review: List[str] = [] + if ambiguous_art: + group_review.append("Artwork selection requires review.") + if not losers: + group_review.append("No losers to consolidate.") + + plans.append( + GroupPlan( + group_id=group_id, + winner_path=winner.path, + losers=losers, + planned_winner_tags=planned_tags, + artwork=artwork_actions, + loser_disposition=dispositions, + playlist_rewrites=playlist_map, + review_flags=group_review, + context_summary={ + "album": sorted([t.path for t in cluster if contexts.get(t.path) == "album"]), + "single": sorted([t.path for t in cluster if contexts.get(t.path) == "single"]), + "unknown": sorted([t.path for t in cluster if contexts.get(t.path) == "unknown"]), + }, + ) + ) + + return ConsolidationPlan(groups=plans, review_flags=review_flags) From 26dd9b7fbe11871a0ff7585e4902daaff998e9ff Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 21 Dec 2025 20:42:30 -0500 Subject: [PATCH 148/606] Add duplicate consolidation preview workflow --- duplicate_consolidation.py | 239 +++++++++++++++++- main_gui.py | 228 ++++++++++++++++- tests/test_duplicate_consolidation_preview.py | 64 +++++ 3 files changed, 516 insertions(+), 15 deletions(-) create mode 100644 tests/test_duplicate_consolidation_preview.py diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index 2691d87..38dbc59 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -8,7 +8,10 @@ """ from __future__ import annotations +import datetime import hashlib +import html +import json import os import threading import time @@ -54,7 +57,12 @@ def cover_hash(self) -> str | None: @property def is_lossless(self) -> bool: - return os.path.splitext(self.ext)[1].lower() in LOSSLESS_EXTS + ext = self.ext.lower() + if not ext or ext == ".": + ext = os.path.splitext(self.path)[1].lower() + elif not ext.startswith("."): + ext = f".{ext}" + return ext in LOSSLESS_EXTS @property def metadata_count(self) -> int: @@ -87,6 +95,17 @@ def to_dict(self) -> Dict[str, str]: return {"source": self.source, "target": self.target, "reason": self.reason} +@dataclass +class PlaylistImpact: + """Summary of playlist rewrites driven by a duplicate group.""" + + playlists: int = 0 + entries: int = 0 + + def to_dict(self) -> Dict[str, int]: + return {"playlists": self.playlists, "entries": self.entries} + + @dataclass class GroupPlan: """Planned actions for a duplicate group.""" @@ -95,9 +114,12 @@ class GroupPlan: winner_path: str losers: List[str] planned_winner_tags: Dict[str, object] + metadata_changes: Dict[str, Dict[str, object]] + winner_quality: Dict[str, object] artwork: List[ArtworkDirective] loser_disposition: Dict[str, str] playlist_rewrites: Dict[str, str] + playlist_impact: PlaylistImpact review_flags: List[str] context_summary: Dict[str, List[str]] @@ -107,9 +129,12 @@ def to_dict(self) -> Dict[str, object]: "winner_path": self.winner_path, "losers": list(self.losers), "planned_winner_tags": dict(self.planned_winner_tags), + "metadata_changes": {k: dict(v) for k, v in self.metadata_changes.items()}, + "winner_quality": dict(self.winner_quality), "artwork": [a.to_dict() for a in self.artwork], "loser_disposition": dict(self.loser_disposition), "playlist_rewrites": dict(self.playlist_rewrites), + "playlist_impact": self.playlist_impact.to_dict(), "review_flags": list(self.review_flags), "context_summary": {k: list(v) for k, v in self.context_summary.items()}, } @@ -121,13 +146,25 @@ class ConsolidationPlan: groups: List[GroupPlan] = field(default_factory=list) review_flags: List[str] = field(default_factory=list) + generated_at: datetime.datetime = field(default_factory=lambda: datetime.datetime.now(datetime.timezone.utc)) def to_dict(self) -> Dict[str, object]: return { "groups": [g.to_dict() for g in self.groups], "review_flags": list(self.review_flags), + "generated_at": self.generated_at.isoformat(), } + @property + def review_required_groups(self) -> List[GroupPlan]: + """Return groups that need manual review.""" + return [g for g in self.groups if g.review_flags] + + @property + def review_required_count(self) -> int: + """Return total number of review-required groups.""" + return len(self.review_required_groups) + def _normalize_track(raw: Mapping[str, object]) -> DuplicateTrack: path = str(raw.get("path")) @@ -201,6 +238,59 @@ def _merge_tags(existing: MutableMapping[str, object], source: Mapping[str, obje return merged +def _metadata_changes(winner: DuplicateTrack, planned_tags: Mapping[str, object]) -> Dict[str, Dict[str, object]]: + keys = [ + "artist", + "albumartist", + "title", + "album", + "album_type", + "track", + "tracknumber", + "disc", + "discnumber", + "date", + "year", + "genre", + ] + changes: Dict[str, Dict[str, object]] = {} + for key in keys: + current = winner.tags.get(key) if isinstance(winner.tags, Mapping) else None + planned = planned_tags.get(key) + if (current or "") == (planned or ""): + continue + changes[key] = {"from": current, "to": planned} + return changes + + +def _quality_rationale(winner: DuplicateTrack, runner_up: DuplicateTrack | None, context: str) -> Dict[str, object]: + reasons: List[str] = [] + if winner.is_lossless: + reasons.append("Lossless format") + if runner_up and not runner_up.is_lossless and winner.is_lossless: + reasons.append("Higher-quality format than the next candidate") + if runner_up and winner.bitrate > runner_up.bitrate: + reasons.append("Higher bitrate than the next candidate") + if runner_up and winner.sample_rate > runner_up.sample_rate: + reasons.append("Higher sample rate than the next candidate") + if runner_up and winner.bit_depth > runner_up.bit_depth: + reasons.append("Higher bit depth than the next candidate") + if context == "album": + reasons.append("Album context favored for metadata consistency") + elif context == "single": + reasons.append("Single context favored for artwork alignment") + + return { + "context": context, + "is_lossless": winner.is_lossless, + "bitrate": winner.bitrate, + "sample_rate": winner.sample_rate, + "bit_depth": winner.bit_depth, + "metadata_count": winner.metadata_count, + "reasons": reasons or ["Deterministic ordering by quality"], + } + + def _artwork_score(track: DuplicateTrack) -> tuple: tags = track.tags or {} size_hint = 0 @@ -333,6 +423,7 @@ def build_consolidation_plan( quality_sorted = sorted(cluster, key=lambda t: _quality_tuple(t, contexts[t.path]), reverse=True) winner = quality_sorted[0] losers = [t.path for t in quality_sorted[1:]] + runner_up = quality_sorted[1] if len(quality_sorted) > 1 else None metadata_source = _select_metadata_source(cluster, contexts) planned_tags: Dict[str, object] = dict(winner.tags) @@ -345,6 +436,10 @@ def build_consolidation_plan( else: review_flags.append(f"Missing metadata source for group containing {winner.path}.") + meta_changes = _metadata_changes(winner, planned_tags) + winner_context = contexts.get(winner.path, "unknown") + winner_quality = _quality_rationale(winner, runner_up, winner_context) + single_candidates = [t for t in cluster if contexts.get(t.path) == "single"] best_art, ambiguous_art = _select_artwork_candidate(single_candidates) artwork_actions: List[ArtworkDirective] = [] @@ -366,15 +461,20 @@ def build_consolidation_plan( if not losers: group_review.append("No losers to consolidate.") + playlist_impact = PlaylistImpact(playlists=len(losers), entries=len(losers)) + plans.append( GroupPlan( group_id=group_id, winner_path=winner.path, losers=losers, planned_winner_tags=planned_tags, + metadata_changes=meta_changes, + winner_quality=winner_quality, artwork=artwork_actions, loser_disposition=dispositions, playlist_rewrites=playlist_map, + playlist_impact=playlist_impact, review_flags=group_review, context_summary={ "album": sorted([t.path for t in cluster if contexts.get(t.path) == "album"]), @@ -385,3 +485,140 @@ def build_consolidation_plan( ) return ConsolidationPlan(groups=plans, review_flags=review_flags) + + +def _html_escape(text: object) -> str: + return html.escape("" if text is None else str(text)) + + +def render_consolidation_preview(plan: ConsolidationPlan, output_html_path: str) -> str: + """Render an HTML preview summarizing the consolidation plan.""" + + def _render_metadata(changes: Dict[str, Dict[str, object]]) -> str: + if not changes: + return "No tag changes planned." + rows = [] + for key in sorted(changes.keys()): + diff = changes[key] + rows.append( + f"{_html_escape(key)}{_html_escape(diff.get('from') or '—')}" + f"{_html_escape(diff.get('to') or '—')}" + ) + return ( + "" + "" + + "".join(rows) + + "" + ) + + def _render_artwork(actions: List[ArtworkDirective]) -> str: + if not actions: + return "Winner artwork unchanged." + rows = [ + f"
      • Copy artwork from {_html_escape(act.source)} → {_html_escape(act.target)} " + f"({_html_escape(act.reason)})
      • " + for act in actions + ] + return "
          " + "".join(rows) + "
        " + + review_required = plan.review_required_groups + lines = [ + "", + "

        Duplicate Consolidation Preview

        ", + f"

        Generated at {plan.generated_at.isoformat()}

        ", + f"

        Total groups: {len(plan.groups)} | Review required: {len(review_required)} | " + f"Global flags: {len(plan.review_flags)}

        ", + ] + + if plan.review_flags: + lines.append("
        Plan Warnings
          ") + for flag in plan.review_flags: + lines.append(f"
        • {_html_escape(flag)}
        • ") + lines.append("
        ") + + for group in plan.groups: + review_badge = ( + "Review required" if group.review_flags else "Ready" + ) + lines.append("
        ".format(cls="review" if group.review_flags else "")) + lines.append( + f"
        Group {group.group_id}
        {review_badge}
        " + ) + lines.append(f"

        Winner: {_html_escape(group.winner_path)}

        ") + lines.append("
          ") + for loser in group.losers: + disposition = group.loser_disposition.get(loser, "quarantine") + lines.append( + f"
        • Loser: {_html_escape(loser)} → {_html_escape(disposition)} | " + f"Playlist rewrite → {_html_escape(group.playlist_rewrites.get(loser, 'n/a'))}
        • " + ) + lines.append("
        ") + + lines.append("

        Quality rationale

        ") + reasons = group.winner_quality.get("reasons") or [] + lines.append( + "
          " + + "".join(f"
        • {_html_escape(r)}
        • " for r in reasons) + + "
        " + ) + lines.append("

        Metadata normalization

        ") + lines.append(_render_metadata(group.metadata_changes)) + + lines.append("

        Artwork

        ") + lines.append(_render_artwork(group.artwork)) + + lines.append( + f"

        Playlist impact: {group.playlist_impact.playlists} playlists, " + f"{group.playlist_impact.entries} entries

        " + ) + + if group.review_flags: + lines.append("

        Review flags:

          ") + for flag in group.review_flags: + lines.append(f"
        • {_html_escape(flag)}
        • ") + lines.append("
        ") + + lines.append( + "
        Context
          " + + f"
        • Album tracks: {len(group.context_summary.get('album', []))}
        • " + + f"
        • Singles: {len(group.context_summary.get('single', []))}
        • " + + f"
        • Unknown: {len(group.context_summary.get('unknown', []))}
        • " + + "
        " + ) + lines.append("
        ") + + lines.append("") + + with open(output_html_path, "w", encoding="utf-8") as f: + f.write("\n".join(lines)) + return output_html_path + + +def export_consolidation_preview(plan: ConsolidationPlan, output_json_path: str) -> str: + """Write a JSON audit of the consolidation plan.""" + + summary = { + "groups": len(plan.groups), + "review_required": plan.review_required_count, + "review_flags": plan.review_flags, + } + payload = { + "generated_at": plan.generated_at.isoformat(), + "summary": summary, + "plan": plan.to_dict(), + } + + with open(output_json_path, "w", encoding="utf-8") as f: + json.dump(payload, f, indent=2) + return output_json_path diff --git a/main_gui.py b/main_gui.py index a0b276d..3c28454 100644 --- a/main_gui.py +++ b/main_gui.py @@ -41,6 +41,11 @@ find_duplicates as api_find_duplicates, get_tags, ) +from duplicate_consolidation import ( + build_consolidation_plan, + render_consolidation_preview, + export_consolidation_preview, +) from controllers.library_index_controller import generate_index from gui.audio_preview import PlaybackError, VlcPreviewPlayer from io import BytesIO @@ -1460,6 +1465,10 @@ def __init__(self, parent: tk.Widget, library_path: str): ) self.update_playlists_var = tk.BooleanVar(value=False) self.quarantine_var = tk.BooleanVar(value=True) + self.override_review_var = tk.BooleanVar(value=False) + self.preview_html_path: str | None = None + self.preview_json_path: str | None = None + self._plan = None container = ttk.Frame(self) container.pack(fill="both", expand=True, padx=10, pady=10) @@ -1509,6 +1518,10 @@ def __init__(self, parent: tk.Widget, library_path: str): ttk.Button(controls, text="Preview", command=self._handle_preview).pack( side="left", padx=(0, 6) ) + self.open_preview_btn = ttk.Button( + controls, text="Open Preview", command=self._open_preview_output, state="disabled" + ) + self.open_preview_btn.pack(side="left", padx=(0, 6)) ttk.Button(controls, text="Execute", command=self._handle_execute).pack( side="left", padx=(0, 12) ) @@ -1524,6 +1537,11 @@ def __init__(self, parent: tk.Widget, library_path: str): variable=self.quarantine_var, command=self._toggle_quarantine, ).pack(side="left") + ttk.Checkbutton( + controls, + text="Override review blocks", + variable=self.override_review_var, + ).pack(side="left", padx=(10, 0)) # Progress + status status_frame = ttk.Frame(container) @@ -1565,15 +1583,12 @@ def __init__(self, parent: tk.Widget, library_path: str): width = 90 if cid in ("group", "count") else 160 self.groups_tree.column(cid, width=width, anchor="w") self.groups_tree.pack(fill="both", expand=True, padx=6, pady=6) + self.groups_tree.bind("<>", self._on_group_select) inspector = ttk.LabelFrame(results, text="Group Details") inspector.grid(row=0, column=1, sticky="nsew", padx=(3, 6), pady=6) - ttk.Label( - inspector, - text="Select a group to view details", - justify="center", - foreground="#555", - ).pack(fill="both", expand=True, padx=12, pady=12) + self.group_details = ScrolledText(inspector, height=12, state="disabled", wrap="word") + self.group_details.pack(fill="both", expand=True, padx=8, pady=8) self._log_action("Duplicate Finder shell initialized") @@ -1593,6 +1608,69 @@ def _log_action(self, message: str) -> None: self.log_text.see("end") self.log_text.configure(state="disabled") + def _update_groups_view(self, plan) -> None: + self.groups_tree.delete(*self.groups_tree.get_children()) + for group in plan.groups: + title = plan and group.planned_winner_tags.get("title") if hasattr(group, "planned_winner_tags") else None + status = "Review" if group.review_flags else "Ready" + self.groups_tree.insert( + "", + "end", + iid=group.group_id, + values=(group.group_id, title or os.path.basename(group.winner_path), len(group.losers) + 1, status), + tags=("review",) if group.review_flags else (), + ) + self.groups_tree.tag_configure("review", background="#fff3cd") + self.group_details.configure(state="normal") + self.group_details.delete("1.0", "end") + self.group_details.insert("end", "Select a group to view details.") + self.group_details.configure(state="disabled") + + def _on_group_select(self, event=None) -> None: + sel = self.groups_tree.selection() + if not sel or not self._plan: + return + group_id = sel[0] + group = next((g for g in self._plan.groups if g.group_id == group_id), None) + if group: + self._render_group_details(group) + + def _render_group_details(self, group) -> None: + lines = [ + f"Winner: {group.winner_path}", + f"Losers: {len(group.losers)}", + "Dispositions:", + ] + for loser in group.losers: + disp = group.loser_disposition.get(loser, "quarantine") + playlist = group.playlist_rewrites.get(loser, "n/a") + lines.append(f" - {loser} → {disp} (playlist → {playlist})") + + lines.append("Quality rationale:") + for reason in group.winner_quality.get("reasons") or []: + lines.append(f" • {reason}") + + if group.review_flags: + lines.append("Review flags:") + for flag in group.review_flags: + lines.append(f" ! {flag}") + + lines.append("Tag changes:") + if not group.metadata_changes: + lines.append(" (none)") + else: + for key, diff in sorted(group.metadata_changes.items()): + lines.append(f" {key}: {diff.get('from')} → {diff.get('to')}") + + lines.append( + f"Playlist impact: {group.playlist_impact.playlists} playlists, {group.playlist_impact.entries} entries" + ) + + self.group_details.configure(state="normal") + self.group_details.delete("1.0", "end") + self.group_details.insert("end", "\n".join(lines)) + self.group_details.configure(state="disabled") + def _set_status(self, status: str, progress: float | None = None) -> None: self.status_var.set(status) if progress is not None: @@ -1631,27 +1709,140 @@ def _browse_playlist(self) -> None: self.playlist_path_var.set(chosen) self._log_action(f"Playlist folder updated to {chosen}") - def _handle_scan(self) -> None: + def _demo_tracks(self, library_root: str) -> list[dict[str, object]]: + base = library_root or "/library" + return [ + { + "path": os.path.join(base, "Album", "Track A (Master).flac"), + "fingerprint": "fp-demo-001", + "ext": ".flac", + "bitrate": 1100, + "sample_rate": 48000, + "bit_depth": 24, + "tags": { + "title": "Track A", + "album": "Demo Album", + "album_type": "album", + "track": 1, + "artist": "Demo Artist", + "cover_hash": "cover-a", + }, + }, + { + "path": os.path.join(base, "Album", "Track A (MP3).mp3"), + "fingerprint": "fp-demo-001", + "ext": ".mp3", + "bitrate": 320, + "sample_rate": 44100, + "bit_depth": 0, + "tags": { + "title": "Track A", + "album": "Demo Album", + "track": 1, + "artist": "Demo Artist", + "albumartist": "Demo Artist", + }, + }, + { + "path": os.path.join(base, "Singles", "Track B (Single).m4a"), + "fingerprint": "fp-demo-002", + "ext": ".m4a", + "bitrate": 256, + "sample_rate": 44100, + "bit_depth": 0, + "tags": { + "title": "Track B", + "album": "Track B - Single", + "album_type": "single", + "track": 1, + "artist": "Demo Artist", + "artwork_hash": "art-b", + "genre": "Indie", + }, + }, + { + "path": os.path.join(base, "Singles", "Track B (Lossless).flac"), + "fingerprint": "fp-demo-002", + "ext": ".flac", + "bitrate": 900, + "sample_rate": 44100, + "bit_depth": 24, + "tags": { + "title": "Track B", + "album": "Track B - Single", + "album_type": "single", + "track": 1, + "artist": "Demo Artist", + "cover_hash": "art-b", + "genre": "Indie", + }, + }, + ] + + def _generate_plan(self, write_preview: bool) -> None: path = self._validate_library_root() if not path: return - self._log_action("Scan clicked") - self._set_status("Scanning", progress=25) - self._set_status("Ready", progress=100) - self._log_action("Scan stub completed (no files touched)") - def _handle_preview(self) -> None: + tracks = self._demo_tracks(path) + self._set_status("Building plan…", progress=25) + plan = build_consolidation_plan(tracks) + self._plan = plan + self._update_groups_view(plan) + + docs_dir = os.path.join(path, "Docs") + os.makedirs(docs_dir, exist_ok=True) + if write_preview: + html_path = os.path.join(docs_dir, "duplicate_preview.html") + json_path = os.path.join(docs_dir, "duplicate_preview.json") + render_consolidation_preview(plan, html_path) + export_consolidation_preview(plan, json_path) + self.preview_html_path = html_path + self.preview_json_path = json_path + self.open_preview_btn.config(state="normal") + self._log_action(f"Preview written to {html_path}") + self._log_action(f"Audit JSON written to {json_path}") + self._set_status("Preview generated", progress=100) + else: + self.preview_html_path = None + self.preview_json_path = None + self.open_preview_btn.config(state="disabled") + self._set_status("Plan ready", progress=100) + + self._log_action( + f"Plan: {len(plan.groups)} groups, review required={plan.review_required_count}" + ) + if plan.review_required_count: + self._log_action("Review required groups will block execution unless overridden.") + + def _handle_scan(self) -> None: path = self._validate_library_root() if not path: return + self._log_action("Scan clicked") + self._generate_plan(write_preview=False) + + def _handle_preview(self) -> None: self._log_action("Preview clicked") - self._set_status("Preview generated", progress=100) + self._generate_plan(write_preview=True) def _handle_execute(self) -> None: path = self._validate_library_root() if not path: return - self._log_action("Execute clicked") + if not self._plan: + messagebox.showwarning("Preview Required", "Generate a preview before executing.") + self._log_action("Execute blocked: no plan generated") + return + if self._plan.review_required_count and not self.override_review_var.get(): + messagebox.showwarning( + "Review Required", + "Resolve review-required groups or check Override review blocks to proceed.", + ) + self._log_action("Execute blocked: review required groups pending") + return + + self._log_action("Execute clicked (no mutations; preview reused)") self._set_status("Executed", progress=100) def _toggle_update_playlists(self) -> None: @@ -1662,6 +1853,15 @@ def _toggle_quarantine(self) -> None: state = "enabled" if self.quarantine_var.get() else "disabled" self._log_action(f"Quarantine Duplicates {state}") + def _open_preview_output(self) -> None: + if not self.preview_html_path: + messagebox.showinfo("Preview", "No preview has been generated yet.") + return + if not os.path.exists(self.preview_html_path): + messagebox.showerror("Preview Missing", "Preview file could not be found. Generate a new preview.") + return + webbrowser.open(self.preview_html_path) + class SoundVaultImporterApp(tk.Tk): def __init__(self): super().__init__() diff --git a/tests/test_duplicate_consolidation_preview.py b/tests/test_duplicate_consolidation_preview.py new file mode 100644 index 0000000..0f2574c --- /dev/null +++ b/tests/test_duplicate_consolidation_preview.py @@ -0,0 +1,64 @@ +import json +import os + +from duplicate_consolidation import ( + build_consolidation_plan, + render_consolidation_preview, + export_consolidation_preview, +) + + +def _sample_tracks(root: str) -> list[dict[str, object]]: + return [ + { + "path": os.path.join(root, "Album", "Song A.flac"), + "fingerprint": "fp-test-001", + "ext": ".flac", + "bitrate": 1000, + "sample_rate": 48000, + "bit_depth": 24, + "tags": {"title": "Song A", "album": "Album A", "album_type": "album", "cover_hash": "cover-a"}, + }, + { + "path": os.path.join(root, "Album", "Song A.mp3"), + "fingerprint": "fp-test-001", + "ext": ".mp3", + "bitrate": 320, + "sample_rate": 44100, + "bit_depth": 0, + "tags": { + "title": "Song A", + "album": "Album A", + "albumartist": "Various", + "track": 1, + "discnumber": 1, + }, + }, + ] + + +def test_plan_contains_metadata_and_quality_summary(tmp_path) -> None: + plan = build_consolidation_plan(_sample_tracks(str(tmp_path))) + assert plan.groups + group = plan.groups[0] + + assert group.metadata_changes # winner tags should be merged + assert group.winner_quality["reasons"] # rationale present + assert group.playlist_impact.entries == len(group.losers) + + +def test_preview_render_and_export(tmp_path) -> None: + plan = build_consolidation_plan(_sample_tracks(str(tmp_path))) + html_path = tmp_path / "preview.html" + json_path = tmp_path / "preview.json" + + render_consolidation_preview(plan, str(html_path)) + export_consolidation_preview(plan, str(json_path)) + + html = html_path.read_text() + assert "Duplicate Consolidation Preview" in html + assert plan.groups[0].winner_path in html + + data = json.loads(json_path.read_text()) + assert data["summary"]["groups"] == len(plan.groups) + assert data["summary"]["review_required"] == plan.review_required_count From 226bdc7bea9804bf0f5a02db5db3b932fda54125 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 21 Dec 2025 20:51:32 -0500 Subject: [PATCH 149/606] Add executor for duplicate consolidation operations --- duplicate_consolidation_executor.py | 560 ++++++++++++++++++++++++++++ 1 file changed, 560 insertions(+) create mode 100644 duplicate_consolidation_executor.py diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py new file mode 100644 index 0000000..ab765ae --- /dev/null +++ b/duplicate_consolidation_executor.py @@ -0,0 +1,560 @@ +"""Execute duplicate consolidation plans with safe ordering and reporting. + +This module consumes the dry-run plan produced by ``duplicate_consolidation`` +and performs the requested playlist rewrites, artwork transfers, metadata +normalization, and loser disposition handling (quarantine/delete). The +execution pipeline is intentionally conservative: + +- Playlists are backed up before edits and validated after rewrites. +- A cancellation request stops processing after the current operation. +- Errors stop subsequent steps while preserving any backups already written. +- Every run produces machine-readable and human-readable reports. +""" +from __future__ import annotations + +import datetime +import importlib.util +import json +import os +import shutil +import tempfile +import threading +from dataclasses import dataclass, field +from typing import Callable, Dict, Iterable, List, Mapping, MutableMapping, Sequence + +from duplicate_consolidation import ArtworkDirective, ConsolidationPlan +from indexer_control import IndexCancelled, cancel_event as global_cancel_event +from utils.path_helpers import ensure_long_path + +_MUTAGEN_AVAILABLE = importlib.util.find_spec("mutagen") is not None +if _MUTAGEN_AVAILABLE: + from mutagen import File as MutagenFile # type: ignore +else: # pragma: no cover - optional dependency + MutagenFile = None # type: ignore + + +def _default_log(msg: str) -> None: + return None + + +@dataclass +class ExecutionConfig: + """Configuration for consolidation execution.""" + + library_root: str + reports_dir: str + playlists_dir: str | None = None + quarantine_dir: str | None = None + cancel_event: threading.Event | None = None + log_callback: Callable[[str], None] | None = None + apply_metadata: bool = True + apply_artwork: bool = True + + +@dataclass +class ExecutionAction: + """Individual action recorded in the audit log.""" + + step: str + target: str + status: str + detail: str + metadata: Dict[str, object] = field(default_factory=dict) + + def to_dict(self) -> Dict[str, object]: + return { + "step": self.step, + "target": self.target, + "status": self.status, + "detail": self.detail, + "metadata": dict(self.metadata), + } + + +@dataclass +class PlaylistRewriteResult: + """Outcome of rewriting a single playlist.""" + + playlist: str + backup_path: str + replaced_entries: int + status: str + detail: str = "" + + def to_dict(self) -> Dict[str, object]: + return { + "playlist": self.playlist, + "backup_path": self.backup_path, + "replaced_entries": self.replaced_entries, + "status": self.status, + "detail": self.detail, + } + + +@dataclass +class ExecutionResult: + """Aggregate execution outcome and artifact locations.""" + + success: bool + actions: List[ExecutionAction] + playlist_reports: List[PlaylistRewriteResult] + quarantine_index: Dict[str, str] + report_paths: Dict[str, str] + + +def _timestamped_dir(base: str) -> str: + ts = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%d_%H%M%S") + path = os.path.join(base, f"consolidation_run_{ts}") + os.makedirs(path, exist_ok=True) + return path + + +def _atomic_write_text(path: str, content: str, *, encoding: str = "utf-8") -> None: + directory = os.path.dirname(path) or "." + os.makedirs(directory, exist_ok=True) + tmp = tempfile.NamedTemporaryFile(delete=False, dir=directory) + tmp_path = tmp.name + try: + with open(tmp_path, "w", encoding=encoding) as handle: + handle.write(content) + os.replace(tmp_path, path) + finally: + if os.path.exists(tmp_path): + try: + os.unlink(tmp_path) + except OSError: + pass + + +def _atomic_write_json(path: str, payload: Mapping[str, object]) -> None: + text = json.dumps(payload, indent=2, sort_keys=True) + _atomic_write_text(path, text) + + +def _iter_playlists(playlists_dir: str) -> Iterable[str]: + if not os.path.isdir(playlists_dir): + return [] + for dirpath, _dirs, files in os.walk(playlists_dir): + for fname in files: + if fname.lower().endswith(".m3u"): + yield os.path.join(dirpath, fname) + + +def _normalize_playlist_entry(entry: str, playlist_dir: str) -> str: + if not entry or entry.startswith("#"): + return entry + if os.path.isabs(entry): + return os.path.normpath(entry) + return os.path.normpath(os.path.join(playlist_dir, entry)) + + +def _rewrite_playlist(path: str, mapping: Mapping[str, str]) -> tuple[int, List[str]]: + playlist_dir = os.path.dirname(path) + replaced = 0 + try: + with open(path, "r", encoding="utf-8") as handle: + lines = [ln.rstrip("\n") for ln in handle] + except FileNotFoundError: + return 0, [] + + new_lines: List[str] = [] + for ln in lines: + normalized = _normalize_playlist_entry(ln, playlist_dir) + if normalized in mapping: + replaced += 1 + new_abs = mapping[normalized] + rel = os.path.relpath(new_abs, playlist_dir) + new_lines.append(rel) + else: + new_lines.append(ln) + return replaced, new_lines + + +def _write_playlist_atomic(path: str, lines: Sequence[str]) -> None: + directory = os.path.dirname(path) + os.makedirs(directory, exist_ok=True) + tmp = tempfile.NamedTemporaryFile("w", delete=False, dir=directory, encoding="utf-8") + tmp_path = tmp.name + try: + with tmp: + for ln in lines: + tmp.write(ln + "\n") + os.replace(tmp_path, path) + finally: + if os.path.exists(tmp_path): + try: + os.unlink(tmp_path) + except OSError: + pass + + +def _validate_playlist(path: str) -> List[str]: + playlist_dir = os.path.dirname(path) + try: + with open(path, "r", encoding="utf-8") as handle: + lines = [ln.rstrip("\n") for ln in handle] + except FileNotFoundError: + return ["Playlist missing after rewrite."] + + missing: List[str] = [] + for ln in lines: + normalized = _normalize_playlist_entry(ln, playlist_dir) + if not normalized or normalized.startswith("#"): + continue + if not os.path.exists(normalized): + missing.append(normalized) + return missing + + +def _backup_playlist(path: str, backup_root: str) -> str: + rel = os.path.relpath(path, os.path.commonpath([path, backup_root])) + dest = os.path.join(backup_root, rel) + os.makedirs(os.path.dirname(dest), exist_ok=True) + shutil.copy2(path, dest) + return dest + + +def _extract_artwork_bytes(path: str) -> bytes | None: + sidecars = [f"{path}.artwork", f"{path}.cover", f"{path}.jpg"] + for candidate in sidecars: + if os.path.exists(candidate): + with open(candidate, "rb") as handle: + return handle.read() + if MutagenFile is None: + return None + audio = MutagenFile(ensure_long_path(path)) + if audio is None: + return None + pictures = getattr(audio, "pictures", None) + if pictures: + pic = pictures[0] + return bytes(pic.data) if hasattr(pic, "data") else bytes(pic) + tags = getattr(audio, "tags", {}) or {} + for key in list(tags.keys()): + if key.startswith("APIC"): + frame = tags[key] + data = getattr(frame, "data", None) + if data: + return bytes(data) + return None + + +def _apply_artwork(action: ArtworkDirective) -> bool: + payload = _extract_artwork_bytes(action.source) + if payload is None: + return False + sidecar = f"{action.target}.artwork" + directory = os.path.dirname(sidecar) + os.makedirs(directory, exist_ok=True) + tmp = tempfile.NamedTemporaryFile(delete=False, dir=directory) + tmp_path = tmp.name + try: + with open(tmp_path, "wb") as handle: + handle.write(payload) + os.replace(tmp_path, sidecar) + finally: + if os.path.exists(tmp_path): + try: + os.unlink(tmp_path) + except OSError: + pass + if MutagenFile is None: + return True + audio = MutagenFile(ensure_long_path(action.target)) + if audio is None: + return True + try: + pics = getattr(audio, "pictures", None) + if pics is not None: + pics.clear() + pics.append(payload) + elif hasattr(audio, "tags"): + audio.tags["APIC:Consolidation"] = payload + audio.save() + except Exception: + return False + return True + + +def _apply_metadata(target: str, planned_tags: Mapping[str, object]) -> bool: + meta_path = f"{target}.metadata.json" + try: + _atomic_write_json(meta_path, dict(planned_tags)) + except Exception: + return False + if MutagenFile is None: + return True + audio = MutagenFile(ensure_long_path(target), easy=True) + if audio is None: + return True + changed = False + for key, value in planned_tags.items(): + if value is None: + continue + current = audio.tags.get(key) if audio.tags else None + normalized = [str(value)] if not isinstance(value, list) else [str(v) for v in value] + if current != normalized: + audio[key] = normalized + changed = True + if changed: + try: + audio.save() + except Exception: + return False + return True + + +def _quarantine_path(path: str, quarantine_root: str, library_root: str) -> str: + try: + rel = os.path.relpath(path, library_root) + if rel.startswith(".."): + rel = os.path.basename(path) + except ValueError: + rel = os.path.basename(path) + dest = os.path.join(quarantine_root, rel) + os.makedirs(os.path.dirname(dest), exist_ok=True) + return dest + + +def execute_consolidation_plan(plan: ConsolidationPlan, config: ExecutionConfig) -> ExecutionResult: + """Execute the provided consolidation plan with safety guarantees.""" + + cancel = config.cancel_event or global_cancel_event + log = config.log_callback or _default_log + actions: List[ExecutionAction] = [] + playlist_results: List[PlaylistRewriteResult] = [] + quarantine_index: Dict[str, str] = {} + + run_root = _timestamped_dir(config.reports_dir) + reports_dir = os.path.join(run_root, "reports") + backups_dir = os.path.join(run_root, "playlist_backups") + os.makedirs(reports_dir, exist_ok=True) + os.makedirs(backups_dir, exist_ok=True) + + playlists_dir = config.playlists_dir or os.path.join(config.library_root, "Playlists") + quarantine_dir = config.quarantine_dir or os.path.join(config.library_root, "Quarantine") + + loser_to_winner: Dict[str, str] = {} + loser_disposition: Dict[str, str] = {} + artwork_actions: List[ArtworkDirective] = [] + planned_tags: Dict[str, Dict[str, object]] = {} + + for group in plan.groups: + planned_tags[group.winner_path] = dict(group.planned_winner_tags) + for loser in group.losers: + loser_to_winner[loser] = group.winner_path + loser_disposition[loser] = group.loser_disposition.get(loser, "quarantine") + artwork_actions.extend(group.artwork) + + def _record(step: str, target: str, status: str, detail: str, **metadata: object) -> None: + actions.append(ExecutionAction(step=step, target=target, status=status, detail=detail, metadata=metadata)) + + def _check_cancel(stage: str) -> None: + if cancel.is_set(): + _record(stage, "execution", "cancelled", "Execution cancelled by user request.") + raise IndexCancelled() + + success = True + + try: + _check_cancel("start") + + # Step 1: backup playlists that will change + impacted: List[str] = [] + for playlist in _iter_playlists(playlists_dir): + try: + with open(playlist, "r", encoding="utf-8") as handle: + lines = [ln.rstrip("\n") for ln in handle] + except FileNotFoundError: + continue + playlist_dir = os.path.dirname(playlist) + normalized = {_normalize_playlist_entry(ln, playlist_dir) for ln in lines if ln} + if normalized & loser_to_winner.keys(): + impacted.append(playlist) + + for playlist in impacted: + _check_cancel("backup_playlists") + try: + backup_path = _backup_playlist(playlist, backups_dir) + _record("backup_playlists", playlist, "success", "Playlist backed up.", backup_path=backup_path) + except Exception as exc: + success = False + _record("backup_playlists", playlist, "failed", f"Failed to backup playlist: {exc}") + raise + + # Step 2: rewrite playlists + for playlist in impacted: + _check_cancel("rewrite_playlists") + replaced, new_lines = _rewrite_playlist(playlist, loser_to_winner) + if replaced == 0: + playlist_results.append( + PlaylistRewriteResult( + playlist=playlist, + backup_path=_backup_playlist(playlist, backups_dir), + replaced_entries=0, + status="skipped", + detail="No matching loser entries found.", + ) + ) + _record("rewrite_playlists", playlist, "skipped", "No entries required updates.") + continue + try: + _write_playlist_atomic(playlist, new_lines) + playlist_results.append( + PlaylistRewriteResult( + playlist=playlist, + backup_path=os.path.join(backups_dir, os.path.relpath(playlist, playlists_dir)), + replaced_entries=replaced, + status="success", + ) + ) + _record("rewrite_playlists", playlist, "success", "Playlist rewritten.", replaced=replaced) + except Exception as exc: + success = False + _record("rewrite_playlists", playlist, "failed", f"Failed to rewrite playlist: {exc}") + raise + + # Step 3: validate rewritten playlists + for result in playlist_results: + _check_cancel("validate_playlists") + missing = _validate_playlist(result.playlist) + if missing: + success = False + try: + if os.path.exists(result.backup_path): + shutil.copy2(result.backup_path, result.playlist) + finally: + _record( + "validate_playlists", + result.playlist, + "failed", + "Playlist validation failed; restored backup.", + missing=missing, + ) + raise RuntimeError(f"Playlist validation failed for {result.playlist}") + _record("validate_playlists", result.playlist, "success", "Playlist validated.") + + # Step 4: apply artwork transfers + if config.apply_artwork: + for art in artwork_actions: + _check_cancel("artwork") + ok = _apply_artwork(art) + status = "success" if ok else "failed" + if not ok: + success = False + _record("artwork", art.target, status, art.reason, source=art.source) + if not ok: + raise RuntimeError(f"Artwork copy failed for {art.target}") + + # Step 5: apply metadata normalization + if config.apply_metadata: + for target, tags in planned_tags.items(): + _check_cancel("metadata") + ok = _apply_metadata(target, tags) + status = "success" if ok else "failed" + if not ok: + success = False + _record("metadata", target, status, "Metadata normalized.") + if not ok: + raise RuntimeError(f"Metadata normalization failed for {target}") + + # Step 6: quarantine or delete losers + for loser, disposition in loser_disposition.items(): + _check_cancel("loser_cleanup") + if disposition == "delete": + try: + if os.path.exists(loser): + os.remove(loser) + quarantine_index[loser] = "deleted" + _record("loser_cleanup", loser, "success", "Loser deleted.") + except Exception as exc: + success = False + _record("loser_cleanup", loser, "failed", f"Failed to delete loser: {exc}") + raise + else: + dest = _quarantine_path(loser, quarantine_dir, config.library_root) + try: + if os.path.exists(loser): + shutil.move(loser, dest) + quarantine_index[loser] = dest + _record("loser_cleanup", loser, "success", "Loser quarantined.", quarantine_path=dest) + except Exception as exc: + success = False + _record("loser_cleanup", loser, "failed", f"Failed to quarantine loser: {exc}") + raise + except IndexCancelled: + success = False + except Exception: + success = False + finally: + audit_path = os.path.join(reports_dir, "audit.json") + playlist_report_path = os.path.join(reports_dir, "playlist_report.json") + quarantine_index_path = os.path.join(reports_dir, "quarantine_index.json") + html_report_path = os.path.join(reports_dir, "execution_report.html") + + audit_payload = { + "success": success, + "actions": [a.to_dict() for a in actions], + "generated_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), + } + playlist_payload = { + "backups_dir": backups_dir, + "playlists": [p.to_dict() for p in playlist_results], + } + quarantine_payload = dict(quarantine_index) + + try: + _atomic_write_json(audit_path, audit_payload) + _atomic_write_json(playlist_report_path, playlist_payload) + _atomic_write_json(quarantine_index_path, quarantine_payload) + except Exception: + pass + + try: + html_lines = [ + "Duplicate Consolidation Execution", + "

        Execution Report

        ", + f"

        Status: {'success' if success else 'failed'}

        ", + "
          ", + ] + for action in actions: + html_lines.append( + f"
        • {action.step} — {action.target}: {action.status} ({action.detail})" # noqa: E501 + "
        • " + ) + html_lines.append("
        ") + if playlist_results: + html_lines.append("

        Playlist Changes

          ") + for res in playlist_results: + html_lines.append( + f"
        • {res.playlist} → {res.status} (replaced {res.replaced_entries}); " + f"backup: {res.backup_path}
        • " + ) + html_lines.append("
        ") + if quarantine_index: + html_lines.append("

        Quarantined/Deleted

          ") + for loser, dest in quarantine_index.items(): + html_lines.append(f"
        • {loser} → {dest}
        • ") + html_lines.append("
        ") + html_lines.append("") + _atomic_write_text(html_report_path, "\n".join(html_lines)) + except Exception: + pass + + report_paths = { + "audit": audit_path, + "playlist_report": playlist_report_path, + "quarantine_index": quarantine_index_path, + "html_report": html_report_path, + "reports_root": reports_dir, + "backups_root": backups_dir, + } + + return ExecutionResult( + success=success, + actions=actions, + playlist_reports=playlist_results, + quarantine_index=quarantine_index, + report_paths=report_paths, + ) + From 78af370fe01e32ae1cd26bf912a2d42a9be2d03e Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 21 Dec 2025 21:00:55 -0500 Subject: [PATCH 150/606] Hardening duplicate consolidation safeguards --- duplicate_consolidation.py | 39 +++++-- duplicate_consolidation_executor.py | 52 ++++++++- tests/test_duplicate_consolidation_guards.py | 86 +++++++++++++++ ...st_duplicate_consolidation_review_flags.py | 101 ++++++++++++++++++ 4 files changed, 271 insertions(+), 7 deletions(-) create mode 100644 tests/test_duplicate_consolidation_guards.py create mode 100644 tests/test_duplicate_consolidation_review_flags.py diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index 38dbc59..7092c35 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -25,6 +25,16 @@ DEFAULT_MAX_CANDIDATES = 5000 DEFAULT_MAX_COMPARISONS = 50_000 DEFAULT_TIMEOUT_SEC = 15.0 +REVIEW_KEYWORDS = ( + "remix", + "edit", + "version", + "sped up", + "slowed", + "nightcore", + "speed up", + "speed-up", +) def _now() -> float: @@ -219,6 +229,16 @@ def _stable_group_id(paths: Sequence[str]) -> str: return hashlib.sha1(canonical.encode("utf-8")).hexdigest()[:12] +def _has_review_keyword(track: DuplicateTrack) -> bool: + title = "" + tags = track.tags or {} + if isinstance(tags, Mapping): + title = str(tags.get("title") or tags.get("name") or "").lower() + if not title: + title = os.path.splitext(os.path.basename(track.path))[0].lower() + return any(keyword in title for keyword in REVIEW_KEYWORDS) + + def _select_metadata_source(candidates: Sequence[DuplicateTrack], contexts: Mapping[str, str]) -> DuplicateTrack | None: album_candidates = [c for c in candidates if contexts.get(c.path) == "album"] sorted_cands = sorted( @@ -304,12 +324,11 @@ def _artwork_score(track: DuplicateTrack) -> tuple: size_hint, track.bitrate, track.sample_rate, - track.path.lower(), ) def _select_artwork_candidate(candidates: Sequence[DuplicateTrack]) -> tuple[DuplicateTrack | None, bool]: - ranked = sorted(candidates, key=_artwork_score, reverse=True) + ranked = sorted(candidates, key=lambda t: (_artwork_score(t), t.path.lower()), reverse=True) if not ranked: return None, False if len(ranked) == 1: @@ -353,6 +372,12 @@ def _cluster_duplicates( or comparisons >= max_comparisons or (timeout_sec and (_now() - start_time) > timeout_sec) ): + if cancel_event.is_set() and "Consolidation planning cancelled or timed out during grouping." not in review_flags: + review_flags.append("Consolidation planning cancelled or timed out during grouping.") + elif comparisons >= max_comparisons and "Comparison budget reached; grouping may be incomplete." not in review_flags: + review_flags.append("Comparison budget reached; grouping may be incomplete.") + elif timeout_sec and (_now() - start_time) > timeout_sec and "Consolidation planning timed out while grouping." not in review_flags: + review_flags.append("Consolidation planning timed out while grouping.") break if track.path in used: continue @@ -446,10 +471,10 @@ def build_consolidation_plan( if best_art and (not winner.cover_hash or winner.cover_hash != best_art.cover_hash): reason = "Copy single artwork to preserve release look" artwork_actions.append(ArtworkDirective(source=best_art.path, target=winner.path, reason=reason)) - if ambiguous_art: - review_flags.append( - f"Artwork selection ambiguous for group {_stable_group_id([t.path for t in cluster])}." - ) + if ambiguous_art: + review_flags.append(f"Artwork selection ambiguous for group {_stable_group_id([t.path for t in cluster])}.") + if not best_art and not winner.cover_hash: + review_flags.append(f"No artwork available for group {_stable_group_id([t.path for t in cluster])}.") dispositions = {loser: "quarantine" for loser in losers} playlist_map = {loser: winner.path for loser in losers} @@ -460,6 +485,8 @@ def build_consolidation_plan( group_review.append("Artwork selection requires review.") if not losers: group_review.append("No losers to consolidate.") + if any(_has_review_keyword(t) for t in cluster): + group_review.append("Contains remix/sped-up variant indicators; review recommended.") playlist_impact = PlaylistImpact(playlists=len(losers), entries=len(losers)) diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index ab765ae..b205d15 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -49,6 +49,11 @@ class ExecutionConfig: log_callback: Callable[[str], None] | None = None apply_metadata: bool = True apply_artwork: bool = True + allow_review_required: bool = False + operation_limit: int | None = 500 + confirm_operation_overage: bool = False + allow_deletion: bool = False + dry_run_execute: bool = False @dataclass @@ -359,6 +364,42 @@ def _check_cancel(stage: str) -> None: try: _check_cancel("start") + if plan.review_required_count and not config.allow_review_required: + success = False + _record( + "preflight", + "execution", + "blocked", + "Review-required groups present; enable override to continue.", + review_required=plan.review_required_count, + ) + raise RuntimeError("Execution blocked: review-required groups present.") + + planned_operations = len(loser_to_winner) + len(artwork_actions) + len(planned_tags) + if config.operation_limit and planned_operations > config.operation_limit and not config.confirm_operation_overage: + success = False + _record( + "preflight", + "execution", + "blocked", + "Planned operations exceed configured limit.", + planned_operations=planned_operations, + operation_limit=config.operation_limit, + ) + raise RuntimeError("Execution blocked: operation limit exceeded.") + + has_deletions = any(disposition == "delete" for disposition in loser_disposition.values()) + if has_deletions and not (config.allow_deletion or config.dry_run_execute): + success = False + _record( + "preflight", + "execution", + "blocked", + "Deletion requested but confirmation not provided.", + deletions_requested=sum(1 for d in loser_disposition.values() if d == "delete"), + ) + raise RuntimeError("Execution blocked: deletions require confirmation.") + # Step 1: backup playlists that will change impacted: List[str] = [] for playlist in _iter_playlists(playlists_dir): @@ -461,6 +502,16 @@ def _check_cancel(stage: str) -> None: # Step 6: quarantine or delete losers for loser, disposition in loser_disposition.items(): _check_cancel("loser_cleanup") + if config.dry_run_execute: + quarantine_index[loser] = "retained" + _record( + "loser_cleanup", + loser, + "skipped", + "Dry-run execute enabled; loser not moved or deleted.", + disposition=disposition, + ) + continue if disposition == "delete": try: if os.path.exists(loser): @@ -557,4 +608,3 @@ def _check_cancel(stage: str) -> None: quarantine_index=quarantine_index, report_paths=report_paths, ) - diff --git a/tests/test_duplicate_consolidation_guards.py b/tests/test_duplicate_consolidation_guards.py new file mode 100644 index 0000000..166ec0b --- /dev/null +++ b/tests/test_duplicate_consolidation_guards.py @@ -0,0 +1,86 @@ +import os +import threading + +from duplicate_consolidation import PlaylistImpact, ConsolidationPlan, GroupPlan +from duplicate_consolidation_executor import ExecutionConfig, execute_consolidation_plan + + +def _make_group(tmp_path, *, review_flags=None, disposition="quarantine"): + winner = tmp_path / "winner.flac" + loser = tmp_path / "loser.mp3" + winner.write_text("winner") + loser.write_text("loser") + return GroupPlan( + group_id="g1", + winner_path=str(winner), + losers=[str(loser)], + planned_winner_tags={"title": "Winner"}, + metadata_changes={"title": {"from": "old", "to": "Winner"}}, + winner_quality={"reasons": ["test"]}, + artwork=[], + loser_disposition={str(loser): disposition}, + playlist_rewrites={str(loser): str(winner)}, + playlist_impact=PlaylistImpact(playlists=1, entries=1), + review_flags=review_flags or [], + context_summary={"album": [], "single": [], "unknown": [str(winner)]}, + ) + + +def _config(tmp_path, **overrides): + defaults = dict( + library_root=str(tmp_path), + reports_dir=str(tmp_path / "reports"), + playlists_dir=str(tmp_path / "Playlists"), + quarantine_dir=str(tmp_path / "Quarantine"), + cancel_event=threading.Event(), + log_callback=lambda _m: None, + ) + defaults.update(overrides) + return ExecutionConfig(**defaults) + + +def test_execution_blocks_review_required_groups(tmp_path): + plan = ConsolidationPlan(groups=[_make_group(tmp_path, review_flags=["manual review required"])]) + result = execute_consolidation_plan(plan, _config(tmp_path)) + + assert result.success is False + assert any(action.status == "blocked" for action in result.actions) + assert (tmp_path / "loser.mp3").exists() + + +def test_execution_enforces_operation_limit(tmp_path): + plan = ConsolidationPlan(groups=[_make_group(tmp_path, review_flags=[])]) + cfg = _config(tmp_path, operation_limit=1, confirm_operation_overage=False, allow_review_required=True) + + result = execute_consolidation_plan(plan, cfg) + + assert result.success is False + assert any(action.status == "blocked" and "Planned operations" in action.detail for action in result.actions) + assert (tmp_path / "loser.mp3").exists() + + +def test_execution_requires_deletion_confirmation(tmp_path): + plan = ConsolidationPlan(groups=[_make_group(tmp_path, disposition="delete", review_flags=[])]) + cfg = _config(tmp_path, allow_review_required=True) + + result = execute_consolidation_plan(plan, cfg) + + assert result.success is False + assert (tmp_path / "loser.mp3").exists() + assert any("Deletion requested" in action.detail for action in result.actions) + + +def test_dry_run_execute_skips_loser_cleanup(tmp_path): + plan = ConsolidationPlan(groups=[_make_group(tmp_path, disposition="delete", review_flags=[])]) + cfg = _config( + tmp_path, + allow_review_required=True, + allow_deletion=False, + dry_run_execute=True, + ) + + result = execute_consolidation_plan(plan, cfg) + + assert result.success is True + assert result.quarantine_index[str(tmp_path / "loser.mp3")] == "retained" + assert (tmp_path / "loser.mp3").exists() diff --git a/tests/test_duplicate_consolidation_review_flags.py b/tests/test_duplicate_consolidation_review_flags.py new file mode 100644 index 0000000..ce321ff --- /dev/null +++ b/tests/test_duplicate_consolidation_review_flags.py @@ -0,0 +1,101 @@ +import os +import threading + +from duplicate_consolidation import build_consolidation_plan + + +def test_remix_keywords_trigger_review_flag(tmp_path): + tracks = [ + { + "path": os.path.join(tmp_path, "Song (Remix).flac"), + "fingerprint": "fp-keyword-1", + "ext": ".flac", + "bitrate": 1000, + "sample_rate": 48000, + "bit_depth": 24, + "tags": {"title": "Song (Remix)"}, + }, + { + "path": os.path.join(tmp_path, "Song.flac"), + "fingerprint": "fp-keyword-1", + "ext": ".flac", + "bitrate": 900, + "sample_rate": 44100, + "bit_depth": 16, + "tags": {"title": "Song"}, + }, + ] + + plan = build_consolidation_plan(tracks) + assert plan.groups + assert any("remix" in flag.lower() or "speed" in flag.lower() for flag in plan.groups[0].review_flags) + + +def test_missing_artwork_sets_global_review_flag(tmp_path): + tracks = [ + { + "path": os.path.join(tmp_path, "Album", "Track A.flac"), + "fingerprint": "fp-art-1", + "ext": ".flac", + "bitrate": 800, + "sample_rate": 44100, + "bit_depth": 16, + "tags": {"title": "Track A", "album": "Album"}, + }, + { + "path": os.path.join(tmp_path, "Album", "Track A (copy).flac"), + "fingerprint": "fp-art-1", + "ext": ".flac", + "bitrate": 700, + "sample_rate": 44100, + "bit_depth": 16, + "tags": {"title": "Track A", "album": "Album"}, + }, + ] + + plan = build_consolidation_plan(tracks, cancel_event=threading.Event()) + assert plan.review_flags + assert any("No artwork available" in flag for flag in plan.review_flags) + + +def test_cancellation_sets_review_flag(tmp_path): + tracks = [ + {"path": os.path.join(tmp_path, f"Track {idx}.flac"), "fingerprint": "fp-cancel", "ext": ".flac"} + for idx in range(3) + ] + cancel = threading.Event() + cancel.set() + + plan = build_consolidation_plan(tracks, cancel_event=cancel) + + assert "Cancelled before normalization." in plan.review_flags + assert not plan.groups # cancelled before processing any tracks + + +def test_multiple_singles_with_conflicting_artwork_require_review(tmp_path): + tracks = [ + { + "path": os.path.join(tmp_path, "Singles", "Track A.flac"), + "fingerprint": "fp-art-ambiguous", + "ext": ".flac", + "bitrate": 256, + "sample_rate": 44100, + "bit_depth": 16, + "tags": {"album_type": "single", "cover_hash": "cover-a"}, + }, + { + "path": os.path.join(tmp_path, "Singles", "Track A copy.flac"), + "fingerprint": "fp-art-ambiguous", + "ext": ".flac", + "bitrate": 256, + "sample_rate": 44100, + "bit_depth": 16, + "tags": {"album_type": "single", "cover_hash": "cover-b"}, + }, + ] + + plan = build_consolidation_plan(tracks) + group = plan.groups[0] + + assert any("artwork" in flag.lower() for flag in group.review_flags) + assert any("Artwork selection ambiguous" in flag for flag in plan.review_flags) From d6f1a24b2f6203b3a82ac509b419249024f38a5b Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 21 Dec 2025 21:35:04 -0500 Subject: [PATCH 151/606] Implement real duplicate planning metadata and artwork extraction --- duplicate_consolidation.py | 423 +++++++++++++++--- main_gui.py | 125 +++--- tests/test_duplicate_consolidation_preview.py | 4 + ...st_duplicate_consolidation_review_flags.py | 2 +- 4 files changed, 433 insertions(+), 121 deletions(-) diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index 7092c35..ed11b34 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -11,6 +11,8 @@ import datetime import hashlib import html +import importlib.util +import io import json import os import threading @@ -18,6 +20,20 @@ from dataclasses import dataclass, field from typing import Callable, Dict, Iterable, List, Mapping, MutableMapping, Optional, Sequence +from utils.path_helpers import ensure_long_path + +_MUTAGEN_AVAILABLE = importlib.util.find_spec("mutagen") is not None +if _MUTAGEN_AVAILABLE: + from mutagen import File as MutagenFile # type: ignore +else: # pragma: no cover - optional dependency + MutagenFile = None # type: ignore + +_PIL_AVAILABLE = importlib.util.find_spec("PIL") is not None +if _PIL_AVAILABLE: + from PIL import Image # type: ignore +else: # pragma: no cover - optional dependency + Image = None # type: ignore + from near_duplicate_detector import fingerprint_distance LOSSLESS_EXTS = {".flac", ".wav", ".alac", ".ape", ".aiff", ".aif"} @@ -35,6 +51,22 @@ "speed up", "speed-up", ) +PLACEHOLDER_TOKENS = {"demo", "tbd", "placeholder", "sample", "example", "unknown"} +TAG_KEYS = [ + "artist", + "albumartist", + "title", + "album", + "album_type", + "track", + "tracknumber", + "disc", + "discnumber", + "date", + "year", + "genre", + "compilation", +] def _now() -> float: @@ -45,6 +77,197 @@ def _default_progress(_current: int, _total: int, _msg: str) -> None: return None +def _blank_tags() -> Dict[str, object]: + return {key: None for key in TAG_KEYS} + + +def _first_value(value: object) -> object: + if isinstance(value, list): + return _first_value(value[0]) if value else None + return value + + +def _parse_int(value: object) -> int | None: + try: + ivalue = int(str(value).split("/")[0]) + return ivalue + except Exception: + return None + + +def _extract_image_dimensions(payload: bytes) -> tuple[Optional[int], Optional[int]]: + if not payload or not Image: + return None, None + try: + with Image.open(io.BytesIO(payload)) as img: + return img.width, img.height + except Exception: + return None, None + + +def _read_audio_file(path: str): + if MutagenFile is None: + return None, "mutagen unavailable" + try: + return MutagenFile(ensure_long_path(path)), None + except Exception as exc: # pragma: no cover - depends on local files + return None, f"read failed: {exc}" + + +def _normalize_tags_from_audio(audio) -> Dict[str, object]: + if audio is None or not getattr(audio, "tags", None): + return _blank_tags() + + tags = audio.tags + values: Dict[str, object] = _blank_tags() + + def pick(keys: Sequence[str]) -> object: + for key in keys: + if key in tags: + return _first_value(tags.get(key)) + if hasattr(tags, "getall"): + found = tags.getall(key) + if found: + return _first_value(found) + return None + + values["artist"] = pick(["artist", "TPE1"]) + values["albumartist"] = pick(["albumartist", "album artist", "TPE2"]) + values["title"] = pick(["title", "TIT2"]) + values["album"] = pick(["album", "TALB"]) + values["album_type"] = pick(["albumtype", "album_type", "releasetype", "release_type"]) + + track_no = pick(["tracknumber", "track", "TRCK"]) + disc_no = pick(["discnumber", "disc", "TPOS"]) + values["track"] = _parse_int(track_no) or track_no or None + values["tracknumber"] = values["track"] + values["disc"] = _parse_int(disc_no) or disc_no or None + values["discnumber"] = values["disc"] + + date_val = pick(["date", "year", "TDRC"]) + values["date"] = str(date_val) if date_val is not None else None + values["year"] = _parse_int(date_val) or None + + values["genre"] = pick(["genre", "TCON"]) + values["compilation"] = pick(["compilation", "TCMP"]) + return values + + +def _extract_artwork_from_audio(audio, path: str) -> tuple[List[ArtworkCandidate], Optional[str]]: + if audio is None: + return [], "audio not readable" + + candidates: List[ArtworkCandidate] = [] + + def _record(payload: bytes, status: str = "ok") -> None: + if not payload: + return + width, height = _extract_image_dimensions(payload) + candidates.append( + ArtworkCandidate( + path=path, + hash=hashlib.sha256(payload).hexdigest(), + size=len(payload), + width=width, + height=height, + status=status, + bytes=payload, + ) + ) + + if hasattr(audio, "pictures"): + for pic in getattr(audio, "pictures"): + payload = getattr(pic, "data", None) + if payload: + width = getattr(pic, "width", None) + height = getattr(pic, "height", None) + _record(payload, status="ok") + # Override dimensions if provided by mutagen + if candidates[-1].width is None and width: + candidates[-1].width = width + if candidates[-1].height is None and height: + candidates[-1].height = height + if getattr(audio, "tags", None): + try: + apics = audio.tags.getall("APIC") + for apic in apics: + payload = getattr(apic, "data", None) + if payload: + _record(payload, status="ok") + except Exception: + pass + if "covr" in audio.tags: + for cover in audio.tags.get("covr", []): + payload = bytes(cover) if cover else None + if payload: + _record(payload, status="ok") + if "WM/Picture" in audio.tags: + for pic in audio.tags.get("WM/Picture", []): + payload = getattr(pic, "value", None) + if payload: + _record(payload, status="ok") + + if not candidates and getattr(audio, "tags", None): + return [], "no artwork tags" + if not candidates: + return [], "no artwork found" + return candidates, None + + +def _read_tags_and_artwork(path: str, provided_tags: Mapping[str, object] | None) -> tuple[Dict[str, object], List[ArtworkCandidate], Optional[str], Optional[str]]: + audio, error = _read_audio_file(path) + file_tags = _normalize_tags_from_audio(audio) + base = dict(file_tags) + fallback = provided_tags or {} + for key, val in (fallback or {}).items(): + if key in base and base[key] not in (None, "", []): + continue + base[key] = val + artwork, art_error = _extract_artwork_from_audio(audio, path) + if artwork: + base["cover_hash"] = artwork[0].hash + base["artwork_hash"] = artwork[0].hash + return base, artwork, error, art_error + + +def _placeholder_present(tags: Mapping[str, object]) -> bool: + for key, value in tags.items(): + if value is None: + continue + if isinstance(value, (list, tuple)): + vals = value + else: + vals = [value] + for val in vals: + text = str(val).strip().lower() + if any(token in text for token in PLACEHOLDER_TOKENS): + return True + return False + + +@dataclass +class ArtworkCandidate: + """Discovered artwork and its metadata.""" + + path: str + hash: str + size: int + width: Optional[int] + height: Optional[int] + status: str + bytes: bytes | None = None + + def to_dict(self) -> Dict[str, object]: + return { + "path": self.path, + "hash": self.hash, + "size": self.size, + "width": self.width, + "height": self.height, + "status": self.status, + } + + @dataclass class DuplicateTrack: """Normalized representation of a track for consolidation planning.""" @@ -56,13 +279,20 @@ class DuplicateTrack: sample_rate: int = 0 bit_depth: int = 0 tags: Dict[str, object] = field(default_factory=dict) + current_tags: Dict[str, object] = field(default_factory=dict) + artwork: List[ArtworkCandidate] = field(default_factory=list) + context_evidence: List[str] = field(default_factory=list) + metadata_error: Optional[str] = None + artwork_error: Optional[str] = None @property def cover_hash(self) -> str | None: - value = self.tags.get("cover_hash") if isinstance(self.tags, Mapping) else None + value = None + tag_source = self.current_tags if isinstance(self.current_tags, Mapping) else self.tags + value = tag_source.get("cover_hash") if isinstance(tag_source, Mapping) else None if value: return str(value) - art = self.tags.get("artwork_hash") if isinstance(self.tags, Mapping) else None + art = tag_source.get("artwork_hash") if isinstance(tag_source, Mapping) else None return str(art) if art else None @property @@ -76,7 +306,8 @@ def is_lossless(self) -> bool: @property def metadata_count(self) -> int: - if not isinstance(self.tags, Mapping): + tags = self.current_tags if isinstance(self.current_tags, Mapping) else self.tags + if not isinstance(tags, Mapping): return 0 keys = [ "artist", @@ -90,7 +321,7 @@ def metadata_count(self) -> int: "date", "year", ] - return sum(1 for k in keys if self.tags.get(k)) + return sum(1 for k in keys if tags.get(k)) @dataclass @@ -124,14 +355,22 @@ class GroupPlan: winner_path: str losers: List[str] planned_winner_tags: Dict[str, object] + winner_current_tags: Dict[str, object] + current_tags: Dict[str, Dict[str, object]] metadata_changes: Dict[str, Dict[str, object]] winner_quality: Dict[str, object] artwork: List[ArtworkDirective] + artwork_candidates: List[ArtworkCandidate] + chosen_artwork_source: Dict[str, object] + artwork_status: str loser_disposition: Dict[str, str] playlist_rewrites: Dict[str, str] playlist_impact: PlaylistImpact review_flags: List[str] context_summary: Dict[str, List[str]] + context_evidence: Dict[str, List[str]] + tag_source: Optional[str] + placeholders_present: bool def to_dict(self) -> Dict[str, object]: return { @@ -139,14 +378,22 @@ def to_dict(self) -> Dict[str, object]: "winner_path": self.winner_path, "losers": list(self.losers), "planned_winner_tags": dict(self.planned_winner_tags), + "winner_current_tags": dict(self.winner_current_tags), + "current_tags": {k: dict(v) for k, v in self.current_tags.items()}, "metadata_changes": {k: dict(v) for k, v in self.metadata_changes.items()}, "winner_quality": dict(self.winner_quality), "artwork": [a.to_dict() for a in self.artwork], + "artwork_candidates": [a.to_dict() for a in self.artwork_candidates], + "chosen_artwork_source": dict(self.chosen_artwork_source), + "artwork_status": self.artwork_status, "loser_disposition": dict(self.loser_disposition), "playlist_rewrites": dict(self.playlist_rewrites), "playlist_impact": self.playlist_impact.to_dict(), "review_flags": list(self.review_flags), "context_summary": {k: list(v) for k, v in self.context_summary.items()}, + "context_evidence": {k: list(v) for k, v in self.context_evidence.items()}, + "tag_source": self.tag_source, + "placeholders_present": self.placeholders_present, } @@ -157,12 +404,14 @@ class ConsolidationPlan: groups: List[GroupPlan] = field(default_factory=list) review_flags: List[str] = field(default_factory=list) generated_at: datetime.datetime = field(default_factory=lambda: datetime.datetime.now(datetime.timezone.utc)) + placeholders_present: bool = False def to_dict(self) -> Dict[str, object]: return { "groups": [g.to_dict() for g in self.groups], "review_flags": list(self.review_flags), "generated_at": self.generated_at.isoformat(), + "placeholders_present": self.placeholders_present, } @property @@ -179,7 +428,9 @@ def review_required_count(self) -> int: def _normalize_track(raw: Mapping[str, object]) -> DuplicateTrack: path = str(raw.get("path")) ext = os.path.splitext(path)[1].lower() or str(raw.get("ext", "")).lower() - tags = raw.get("tags") if isinstance(raw.get("tags"), Mapping) else {} + provided_tags = raw.get("tags") if isinstance(raw.get("tags"), Mapping) else {} + current_tags, artwork, meta_err, art_err = _read_tags_and_artwork(path, provided_tags) + tags = dict(current_tags) return DuplicateTrack( path=path, fingerprint=raw.get("fingerprint"), @@ -188,29 +439,42 @@ def _normalize_track(raw: Mapping[str, object]) -> DuplicateTrack: sample_rate=int(raw.get("sample_rate") or raw.get("samplerate") or 0), bit_depth=int(raw.get("bit_depth") or raw.get("bitdepth") or 0), tags=dict(tags), + current_tags=dict(current_tags), + artwork=artwork, + metadata_error=meta_err, + artwork_error=art_err, ) -def _classify_context(track: DuplicateTrack) -> str: - tags = track.tags or {} +def _classify_context(track: DuplicateTrack) -> tuple[str, List[str]]: + tags = track.current_tags or track.tags or {} album = str(tags.get("album") or "").strip() album_type = str(tags.get("album_type") or tags.get("release_type") or "").lower() track_no = tags.get("track") or tags.get("tracknumber") disc_no = tags.get("disc") or tags.get("discnumber") + evidence: List[str] = [] + if album_type == "single": - return "single" + evidence.append("Album type tag indicates single") + return "single", evidence if album_type in {"album", "lp"}: - return "album" + evidence.append("Album type tag indicates album/LP") + return "album", evidence if album.lower().endswith(" - single") or "(single" in album.lower(): - return "single" + evidence.append("Album title formatted like a single") + return "single", evidence if album and (track_no or disc_no): - return "album" + evidence.append("Album tag present with track/disc number") + return "album", evidence if not album: - return "single" + evidence.append("No album tag present") + return "single", evidence if track.cover_hash: - return "album" - return "unknown" + evidence.append("Embedded artwork present suggesting album packaging") + return "album", evidence + evidence.append("Insufficient context to classify") + return "unknown", evidence def _quality_tuple(track: DuplicateTrack, context: str) -> tuple: @@ -250,7 +514,9 @@ def _select_metadata_source(candidates: Sequence[DuplicateTrack], contexts: Mapp def _merge_tags(existing: MutableMapping[str, object], source: Mapping[str, object]) -> Dict[str, object]: - merged = dict(existing) + merged = _blank_tags() + for key, value in existing.items(): + merged[key] = value for key, value in source.items(): if key in merged and merged[key]: continue @@ -259,23 +525,10 @@ def _merge_tags(existing: MutableMapping[str, object], source: Mapping[str, obje def _metadata_changes(winner: DuplicateTrack, planned_tags: Mapping[str, object]) -> Dict[str, Dict[str, object]]: - keys = [ - "artist", - "albumartist", - "title", - "album", - "album_type", - "track", - "tracknumber", - "disc", - "discnumber", - "date", - "year", - "genre", - ] + keys = TAG_KEYS changes: Dict[str, Dict[str, object]] = {} for key in keys: - current = winner.tags.get(key) if isinstance(winner.tags, Mapping) else None + current = winner.current_tags.get(key) if isinstance(winner.current_tags, Mapping) else None planned = planned_tags.get(key) if (current or "") == (planned or ""): continue @@ -312,15 +565,13 @@ def _quality_rationale(winner: DuplicateTrack, runner_up: DuplicateTrack | None, def _artwork_score(track: DuplicateTrack) -> tuple: - tags = track.tags or {} size_hint = 0 - art_bytes = tags.get("artwork_bytes") if isinstance(tags, Mapping) else None - if isinstance(art_bytes, (bytes, bytearray)): - size_hint = len(art_bytes) - elif isinstance(art_bytes, int): - size_hint = art_bytes + has_art = False + if track.artwork: + has_art = True + size_hint = max(c.size for c in track.artwork if c.status == "ok") if any(c.status == "ok" for c in track.artwork) else 0 return ( - 1 if track.cover_hash else 0, + 1 if has_art else 0, size_hint, track.bitrate, track.sample_rate, @@ -328,7 +579,8 @@ def _artwork_score(track: DuplicateTrack) -> tuple: def _select_artwork_candidate(candidates: Sequence[DuplicateTrack]) -> tuple[DuplicateTrack | None, bool]: - ranked = sorted(candidates, key=lambda t: (_artwork_score(t), t.path.lower()), reverse=True) + usable = [c for c in candidates if c.artwork] + ranked = sorted(usable, key=lambda t: (_artwork_score(t), t.path.lower()), reverse=True) if not ranked: return None, False if len(ranked) == 1: @@ -443,21 +695,31 @@ def build_consolidation_plan( ) plans: List[GroupPlan] = [] + plan_placeholders = False for cluster in sorted(clusters, key=lambda c: _stable_group_id([t.path for t in c])): - contexts = {t.path: _classify_context(t) for t in cluster} + contexts: Dict[str, str] = {} + context_evidence: Dict[str, List[str]] = {} + for t in cluster: + ctx, evidence = _classify_context(t) + contexts[t.path] = ctx + t.context_evidence = evidence + context_evidence[t.path] = evidence + quality_sorted = sorted(cluster, key=lambda t: _quality_tuple(t, contexts[t.path]), reverse=True) winner = quality_sorted[0] losers = [t.path for t in quality_sorted[1:]] runner_up = quality_sorted[1] if len(quality_sorted) > 1 else None metadata_source = _select_metadata_source(cluster, contexts) - planned_tags: Dict[str, object] = dict(winner.tags) + planned_tags: Dict[str, object] = _merge_tags(_blank_tags(), winner.current_tags) + tag_source: Optional[str] = None if metadata_source: - planned_tags = _merge_tags(planned_tags, metadata_source.tags) + planned_tags = _merge_tags(planned_tags, metadata_source.current_tags) if contexts.get(metadata_source.path) == "album": planned_tags.setdefault("album_type", "album") elif contexts.get(metadata_source.path) == "single": planned_tags.setdefault("album_type", "single") + tag_source = metadata_source.path else: review_flags.append(f"Missing metadata source for group containing {winner.path}.") @@ -465,40 +727,82 @@ def build_consolidation_plan( winner_context = contexts.get(winner.path, "unknown") winner_quality = _quality_rationale(winner, runner_up, winner_context) + group_review: List[str] = [] + single_candidates = [t for t in cluster if contexts.get(t.path) == "single"] - best_art, ambiguous_art = _select_artwork_candidate(single_candidates) + album_candidates = [t for t in cluster if contexts.get(t.path) == "album"] + art_candidates_order = single_candidates or album_candidates or cluster + distinct_hashes = {t.cover_hash for t in art_candidates_order if t.cover_hash} + tag_artwork_conflict = len(distinct_hashes) > 1 + best_art, ambiguous_art = _select_artwork_candidate(art_candidates_order) + ambiguous_art = ambiguous_art or tag_artwork_conflict artwork_actions: List[ArtworkDirective] = [] - if best_art and (not winner.cover_hash or winner.cover_hash != best_art.cover_hash): - reason = "Copy single artwork to preserve release look" - artwork_actions.append(ArtworkDirective(source=best_art.path, target=winner.path, reason=reason)) + chosen_artwork_source: Dict[str, object] = {"path": None, "reason": ""} + artwork_status = "missing" + if best_art and best_art.artwork: + best_blob = next((c for c in best_art.artwork if c.status == "ok"), best_art.artwork[0]) + chosen_artwork_source = { + "path": best_art.path, + "hash": best_blob.hash, + "size": best_blob.size, + "reason": "best artwork candidate", + } + artwork_status = "selected" + if not winner.cover_hash or winner.cover_hash != best_blob.hash: + reason = "Copy artwork from best candidate to winner" + artwork_actions.append(ArtworkDirective(source=best_art.path, target=winner.path, reason=reason)) if ambiguous_art: review_flags.append(f"Artwork selection ambiguous for group {_stable_group_id([t.path for t in cluster])}.") - if not best_art and not winner.cover_hash: - review_flags.append(f"No artwork available for group {_stable_group_id([t.path for t in cluster])}.") + if not best_art: + missing_reason = "No artwork candidates available" + if any(t.artwork_error for t in cluster): + missing_reason = "Artwork present but unreadable" + review_flags.append(f"{missing_reason} for group {_stable_group_id([t.path for t in cluster])}.") + chosen_artwork_source["reason"] = missing_reason + artwork_status = "none found" + group_review.append(missing_reason) dispositions = {loser: "quarantine" for loser in losers} playlist_map = {loser: winner.path for loser in losers} group_id = _stable_group_id([t.path for t in cluster]) - group_review: List[str] = [] if ambiguous_art: group_review.append("Artwork selection requires review.") if not losers: group_review.append("No losers to consolidate.") if any(_has_review_keyword(t) for t in cluster): group_review.append("Contains remix/sped-up variant indicators; review recommended.") + placeholders = _placeholder_present(planned_tags) or any(_placeholder_present(t.current_tags) for t in cluster) + if placeholders: + group_review.append("Placeholder metadata detected; requires review.") + plan_placeholders = True + if winner.metadata_error: + group_review.append(f"Metadata read issue for winner: {winner.metadata_error}") + if any(t.artwork_error for t in cluster): + group_review.append("Artwork extraction failed for at least one track.") playlist_impact = PlaylistImpact(playlists=len(losers), entries=len(losers)) + artwork_candidates: List[ArtworkCandidate] = [] + for t in cluster: + artwork_candidates.extend(t.artwork) + + current_tags = {t.path: _merge_tags(_blank_tags(), t.current_tags) for t in cluster} + plans.append( GroupPlan( group_id=group_id, winner_path=winner.path, losers=losers, planned_winner_tags=planned_tags, + winner_current_tags=_merge_tags(_blank_tags(), winner.current_tags), + current_tags=current_tags, metadata_changes=meta_changes, winner_quality=winner_quality, artwork=artwork_actions, + artwork_candidates=artwork_candidates, + chosen_artwork_source=chosen_artwork_source, + artwork_status=artwork_status, loser_disposition=dispositions, playlist_rewrites=playlist_map, playlist_impact=playlist_impact, @@ -508,10 +812,16 @@ def build_consolidation_plan( "single": sorted([t.path for t in cluster if contexts.get(t.path) == "single"]), "unknown": sorted([t.path for t in cluster if contexts.get(t.path) == "unknown"]), }, + context_evidence=context_evidence, + tag_source=tag_source, + placeholders_present=placeholders, ) ) - return ConsolidationPlan(groups=plans, review_flags=review_flags) + if plan_placeholders and "Placeholder metadata detected; review required." not in review_flags: + review_flags.append("Placeholder metadata detected; review required.") + + return ConsolidationPlan(groups=plans, review_flags=review_flags, placeholders_present=plan_placeholders) def _html_escape(text: object) -> str: @@ -604,6 +914,10 @@ def _render_artwork(actions: List[ArtworkDirective]) -> str: lines.append("

        Artwork

        ") lines.append(_render_artwork(group.artwork)) + art_source = group.chosen_artwork_source or {} + source_path = art_source.get("path") or "None" + source_reason = art_source.get("reason") or group.artwork_status + lines.append(f"

        Chosen artwork source: {_html_escape(source_path)} ({_html_escape(source_reason)})

        ") lines.append( f"

        Playlist impact: {group.playlist_impact.playlists} playlists, " @@ -621,7 +935,16 @@ def _render_artwork(actions: List[ArtworkDirective]) -> str: + f"

      • Album tracks: {len(group.context_summary.get('album', []))}
      • " + f"
      • Singles: {len(group.context_summary.get('single', []))}
      • " + f"
      • Unknown: {len(group.context_summary.get('unknown', []))}
      • " - + "
      " + + "
    " + + "
      " + + "".join( + f"
    • {_html_escape(path)}: " + + "; ".join(_html_escape(ev) for ev in group.context_evidence.get(path, [])) + + "
    • " + for path in sorted(group.context_evidence.keys()) + ) + + "
    " + + "" ) lines.append("") diff --git a/main_gui.py b/main_gui.py index 3c28454..377098d 100644 --- a/main_gui.py +++ b/main_gui.py @@ -51,6 +51,8 @@ from io import BytesIO from PIL import Image, ImageTk from mutagen import File as MutagenFile +from fingerprint_cache import get_fingerprint +from simple_duplicate_finder import SUPPORTED_EXTS, _compute_fp from tag_fixer import MIN_INTERACTIVE_SCORE, FileRecord from typing import Any, Callable, List from indexer_control import cancel_event, IndexCancelled @@ -1709,82 +1711,65 @@ def _browse_playlist(self) -> None: self.playlist_path_var.set(chosen) self._log_action(f"Playlist folder updated to {chosen}") - def _demo_tracks(self, library_root: str) -> list[dict[str, object]]: - base = library_root or "/library" - return [ - { - "path": os.path.join(base, "Album", "Track A (Master).flac"), - "fingerprint": "fp-demo-001", - "ext": ".flac", - "bitrate": 1100, - "sample_rate": 48000, - "bit_depth": 24, - "tags": { - "title": "Track A", - "album": "Demo Album", - "album_type": "album", - "track": 1, - "artist": "Demo Artist", - "cover_hash": "cover-a", - }, - }, - { - "path": os.path.join(base, "Album", "Track A (MP3).mp3"), - "fingerprint": "fp-demo-001", - "ext": ".mp3", - "bitrate": 320, - "sample_rate": 44100, - "bit_depth": 0, - "tags": { - "title": "Track A", - "album": "Demo Album", - "track": 1, - "artist": "Demo Artist", - "albumartist": "Demo Artist", - }, - }, - { - "path": os.path.join(base, "Singles", "Track B (Single).m4a"), - "fingerprint": "fp-demo-002", - "ext": ".m4a", - "bitrate": 256, - "sample_rate": 44100, - "bit_depth": 0, - "tags": { - "title": "Track B", - "album": "Track B - Single", - "album_type": "single", - "track": 1, - "artist": "Demo Artist", - "artwork_hash": "art-b", - "genre": "Indie", - }, - }, - { - "path": os.path.join(base, "Singles", "Track B (Lossless).flac"), - "fingerprint": "fp-demo-002", - "ext": ".flac", - "bitrate": 900, - "sample_rate": 44100, - "bit_depth": 24, - "tags": { - "title": "Track B", - "album": "Track B - Single", - "album_type": "single", - "track": 1, - "artist": "Demo Artist", - "cover_hash": "art-b", - "genre": "Indie", - }, - }, - ] + def _gather_tracks(self, library_root: str) -> list[dict[str, object]]: + if not library_root: + return [] + docs_dir = os.path.join(library_root, "Docs") + os.makedirs(docs_dir, exist_ok=True) + db_path = os.path.join(docs_dir, ".duplicate_fingerprints.db") + + audio_paths: list[str] = [] + for dirpath, _dirs, files in os.walk(library_root): + rel = os.path.relpath(dirpath, library_root) + parts = {p.lower() for p in rel.split(os.sep)} + if {"not sorted", "playlists"} & parts: + continue + for fname in files: + ext = os.path.splitext(fname)[1].lower() + if ext in SUPPORTED_EXTS: + audio_paths.append(os.path.join(dirpath, fname)) + + tracks: list[dict[str, object]] = [] + total = len(audio_paths) or 1 + for idx, path in enumerate(sorted(audio_paths), start=1): + fp = get_fingerprint(path, db_path, _compute_fp, log_callback=self._log_action) + if not fp: + continue + bitrate = 0 + sample_rate = 0 + bit_depth = 0 + try: + audio = MutagenFile(path) + info = getattr(audio, "info", None) + if info: + bitrate = int(getattr(info, "bitrate", 0) or 0) + sample_rate = int(getattr(info, "sample_rate", 0) or getattr(info, "samplerate", 0) or 0) + bit_depth = int(getattr(info, "bits_per_sample", 0) or getattr(info, "bitdepth", 0) or 0) + except Exception: + pass + tracks.append( + { + "path": path, + "fingerprint": fp, + "ext": os.path.splitext(path)[1].lower(), + "bitrate": bitrate, + "sample_rate": sample_rate, + "bit_depth": bit_depth, + } + ) + self._set_status("Scanning…", progress=min(90, int(idx / total * 70) + 20)) + return tracks def _generate_plan(self, write_preview: bool) -> None: path = self._validate_library_root() if not path: return - tracks = self._demo_tracks(path) + tracks = self._gather_tracks(path) + if not tracks: + messagebox.showwarning("No Tracks", "No audio tracks were found in the selected library.") + self._set_status("Idle", progress=0) + return self._set_status("Building plan…", progress=25) plan = build_consolidation_plan(tracks) self._plan = plan diff --git a/tests/test_duplicate_consolidation_preview.py b/tests/test_duplicate_consolidation_preview.py index 0f2574c..da27efe 100644 --- a/tests/test_duplicate_consolidation_preview.py +++ b/tests/test_duplicate_consolidation_preview.py @@ -45,6 +45,10 @@ def test_plan_contains_metadata_and_quality_summary(tmp_path) -> None: assert group.metadata_changes # winner tags should be merged assert group.winner_quality["reasons"] # rationale present assert group.playlist_impact.entries == len(group.losers) + assert group.winner_current_tags # real tags captured + assert group.current_tags[group.winner_path]["title"] == "Song A" + assert group.chosen_artwork_source # artwork decision captured even if unchanged + assert group.tag_source is not None def test_preview_render_and_export(tmp_path) -> None: diff --git a/tests/test_duplicate_consolidation_review_flags.py b/tests/test_duplicate_consolidation_review_flags.py index ce321ff..704961c 100644 --- a/tests/test_duplicate_consolidation_review_flags.py +++ b/tests/test_duplicate_consolidation_review_flags.py @@ -55,7 +55,7 @@ def test_missing_artwork_sets_global_review_flag(tmp_path): plan = build_consolidation_plan(tracks, cancel_event=threading.Event()) assert plan.review_flags - assert any("No artwork available" in flag for flag in plan.review_flags) + assert any("artwork" in flag.lower() for flag in plan.review_flags) def test_cancellation_sets_review_flag(tmp_path): From 6e5c2b265ee79fec5c5211058ccb257e38f8f1e5 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 21 Dec 2025 21:52:20 -0500 Subject: [PATCH 152/606] Improve duplicate consolidation preview and policies --- duplicate_consolidation.py | 563 ++++++++++++++---- duplicate_consolidation_executor.py | 5 +- .../indexer_engine/music_indexer_api.py | 17 +- music_indexer_api.py | 17 +- tests/test_duplicate_consolidation_guards.py | 13 + tests/test_duplicate_consolidation_preview.py | 118 ++++ 6 files changed, 621 insertions(+), 112 deletions(-) diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index ed11b34..03270ab 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -21,8 +21,11 @@ from typing import Callable, Dict, Iterable, List, Mapping, MutableMapping, Optional, Sequence from utils.path_helpers import ensure_long_path +try: + _MUTAGEN_AVAILABLE = importlib.util.find_spec("mutagen") is not None +except ValueError: # pragma: no cover - defensive: broken mutagen installs + _MUTAGEN_AVAILABLE = False -_MUTAGEN_AVAILABLE = importlib.util.find_spec("mutagen") is not None if _MUTAGEN_AVAILABLE: from mutagen import File as MutagenFile # type: ignore else: # pragma: no cover - optional dependency @@ -214,7 +217,78 @@ def _record(payload: bytes, status: str = "ok") -> None: return candidates, None -def _read_tags_and_artwork(path: str, provided_tags: Mapping[str, object] | None) -> tuple[Dict[str, object], List[ArtworkCandidate], Optional[str], Optional[str]]: +def _coerce_int(value: object) -> int | None: + try: + return int(float(value)) + except (TypeError, ValueError): + return None + + +def _extract_audio_properties(audio, path: str, provided_tags: Mapping[str, object], ext: str) -> Dict[str, object]: + info = getattr(audio, "info", None) + + def pick_int(*candidates: object) -> int: + for candidate in candidates: + if candidate in (None, "", []): + continue + coerced = _coerce_int(candidate) + if coerced is not None: + return coerced + return 0 + + container = ( + str(provided_tags.get("container") or provided_tags.get("format") or ext or "") + .replace(".", "") + .upper() + ) + if not container and audio is not None: + container = audio.__class__.__name__.upper() + + codec = str( + provided_tags.get("codec") + or provided_tags.get("codec_name") + or getattr(info, "codec_name", "") + or getattr(info, "codec", "") + ) + if not codec and hasattr(info, "pprint"): + codec = str(info.pprint()).split(",")[0] + if not codec: + codec = container or os.path.splitext(path)[1].replace(".", "").upper() + + bitrate = pick_int( + provided_tags.get("bitrate"), + getattr(info, "bitrate", None), + getattr(info, "bit_rate", None), + ) + sample_rate = pick_int( + provided_tags.get("sample_rate"), + provided_tags.get("samplerate"), + getattr(info, "sample_rate", None), + getattr(info, "samplerate", None), + ) + bit_depth = pick_int( + provided_tags.get("bit_depth"), + provided_tags.get("bitdepth"), + getattr(info, "bits_per_sample", None), + getattr(info, "bitdepth", None), + ) + channels = pick_int( + provided_tags.get("channels"), + getattr(info, "channels", None), + getattr(info, "channel", None), + ) + + return { + "container": container, + "codec": codec, + "bitrate": bitrate, + "sample_rate": sample_rate, + "bit_depth": bit_depth, + "channels": channels, + } + + +def _read_tags_and_artwork(path: str, provided_tags: Mapping[str, object] | None) -> tuple[Dict[str, object], List[ArtworkCandidate], Optional[str], Optional[str], Dict[str, object]]: audio, error = _read_audio_file(path) file_tags = _normalize_tags_from_audio(audio) base = dict(file_tags) @@ -227,7 +301,8 @@ def _read_tags_and_artwork(path: str, provided_tags: Mapping[str, object] | None if artwork: base["cover_hash"] = artwork[0].hash base["artwork_hash"] = artwork[0].hash - return base, artwork, error, art_error + audio_props = _extract_audio_properties(audio, path, provided_tags or {}, os.path.splitext(path)[1]) + return base, artwork, error, art_error, audio_props def _placeholder_present(tags: Mapping[str, object]) -> bool: @@ -278,6 +353,9 @@ class DuplicateTrack: bitrate: int = 0 sample_rate: int = 0 bit_depth: int = 0 + channels: int = 0 + codec: str = "" + container: str = "" tags: Dict[str, object] = field(default_factory=dict) current_tags: Dict[str, object] = field(default_factory=dict) artwork: List[ArtworkCandidate] = field(default_factory=list) @@ -371,6 +449,11 @@ class GroupPlan: context_evidence: Dict[str, List[str]] tag_source: Optional[str] placeholders_present: bool + tag_source_reason: str + tag_source_evidence: List[str] + track_quality: Dict[str, Dict[str, object]] + group_confidence: str + artwork_evidence: List[str] def to_dict(self) -> Dict[str, object]: return { @@ -394,6 +477,11 @@ def to_dict(self) -> Dict[str, object]: "context_evidence": {k: list(v) for k, v in self.context_evidence.items()}, "tag_source": self.tag_source, "placeholders_present": self.placeholders_present, + "tag_source_reason": self.tag_source_reason, + "tag_source_evidence": list(self.tag_source_evidence), + "track_quality": {k: dict(v) for k, v in self.track_quality.items()}, + "group_confidence": self.group_confidence, + "artwork_evidence": list(self.artwork_evidence), } @@ -429,15 +517,39 @@ def _normalize_track(raw: Mapping[str, object]) -> DuplicateTrack: path = str(raw.get("path")) ext = os.path.splitext(path)[1].lower() or str(raw.get("ext", "")).lower() provided_tags = raw.get("tags") if isinstance(raw.get("tags"), Mapping) else {} - current_tags, artwork, meta_err, art_err = _read_tags_and_artwork(path, provided_tags) + current_tags, artwork, meta_err, art_err, audio_props = _read_tags_and_artwork(path, provided_tags) + provided_artwork: List[ArtworkCandidate] = [] + for art in raw.get("artwork", []) if isinstance(raw.get("artwork"), list) else []: + try: + provided_artwork.append( + ArtworkCandidate( + path=path, + hash=str(art.get("hash")), + size=int(art.get("size") or 0), + width=art.get("width"), + height=art.get("height"), + status=art.get("status", "ok"), + bytes=art.get("bytes"), + ) + ) + except Exception: + continue + if provided_artwork and not artwork: + artwork = provided_artwork + if "cover_hash" not in current_tags: + current_tags["cover_hash"] = provided_artwork[0].hash + current_tags["artwork_hash"] = provided_artwork[0].hash tags = dict(current_tags) return DuplicateTrack( path=path, fingerprint=raw.get("fingerprint"), ext=ext, - bitrate=int(raw.get("bitrate") or 0), - sample_rate=int(raw.get("sample_rate") or raw.get("samplerate") or 0), - bit_depth=int(raw.get("bit_depth") or raw.get("bitdepth") or 0), + bitrate=int(audio_props.get("bitrate") or raw.get("bitrate") or 0), + sample_rate=int(audio_props.get("sample_rate") or raw.get("sample_rate") or raw.get("samplerate") or 0), + bit_depth=int(audio_props.get("bit_depth") or raw.get("bit_depth") or raw.get("bitdepth") or 0), + channels=int(audio_props.get("channels") or raw.get("channels") or 0), + codec=str(audio_props.get("codec") or raw.get("codec") or ""), + container=str(audio_props.get("container") or raw.get("container") or ext.replace(".", "")), tags=dict(tags), current_tags=dict(current_tags), artwork=artwork, @@ -483,7 +595,7 @@ def _quality_tuple(track: DuplicateTrack, context: str) -> tuple: track.bitrate, track.sample_rate, track.bit_depth, - 1 if context == "single" else 0, + track.channels, track.path.lower(), ) @@ -524,6 +636,48 @@ def _merge_tags(existing: MutableMapping[str, object], source: Mapping[str, obje return merged +def _build_planned_tags( + winner: DuplicateTrack, + metadata_source: DuplicateTrack | None, + contexts: Mapping[str, str], +) -> tuple[Dict[str, object], Optional[str], str, List[str]]: + planned_tags: Dict[str, object] = _merge_tags(_blank_tags(), winner.current_tags) + tag_source: Optional[str] = None + tag_source_reason = "Winner tags retained (no metadata source found)" + tag_source_evidence: List[str] = [] + + if metadata_source: + tag_source = metadata_source.path + src_context = contexts.get(metadata_source.path, "unknown") + tag_source_reason = f"Selected {src_context} metadata source with {metadata_source.metadata_count} populated fields" + src_tags = metadata_source.current_tags if isinstance(metadata_source.current_tags, Mapping) else {} + normalized_keys = [ + "album", + "albumartist", + "artist", + "album_type", + "track", + "tracknumber", + "disc", + "discnumber", + "date", + "year", + ] + for key in normalized_keys: + if src_tags.get(key) in (None, "", []): + continue + planned_tags[key] = src_tags.get(key) + if src_context == "album": + planned_tags["album_type"] = src_tags.get("album_type") or "album" + elif src_context == "single": + planned_tags["album_type"] = src_tags.get("album_type") or "single" + tag_source_evidence.append(tag_source_reason) + else: + tag_source_evidence.append("No album-context source available; keeping winner metadata") + + return planned_tags, tag_source, tag_source_reason, tag_source_evidence + + def _metadata_changes(winner: DuplicateTrack, planned_tags: Mapping[str, object]) -> Dict[str, Dict[str, object]]: keys = TAG_KEYS changes: Dict[str, Dict[str, object]] = {} @@ -538,20 +692,37 @@ def _metadata_changes(winner: DuplicateTrack, planned_tags: Mapping[str, object] def _quality_rationale(winner: DuplicateTrack, runner_up: DuplicateTrack | None, context: str) -> Dict[str, object]: reasons: List[str] = [] + + def fmt_codec(t: DuplicateTrack) -> str: + container = t.container or t.ext.replace(".", "").upper() + codec = t.codec or container + return f"{container} / {codec}" + + reasons.append(f"Container/codec: {fmt_codec(winner)}") if winner.is_lossless: - reasons.append("Lossless format") - if runner_up and not runner_up.is_lossless and winner.is_lossless: - reasons.append("Higher-quality format than the next candidate") - if runner_up and winner.bitrate > runner_up.bitrate: - reasons.append("Higher bitrate than the next candidate") - if runner_up and winner.sample_rate > runner_up.sample_rate: - reasons.append("Higher sample rate than the next candidate") - if runner_up and winner.bit_depth > runner_up.bit_depth: - reasons.append("Higher bit depth than the next candidate") - if context == "album": - reasons.append("Album context favored for metadata consistency") - elif context == "single": - reasons.append("Single context favored for artwork alignment") + reasons.append("Lossless encoding") + if runner_up and winner.is_lossless and not runner_up.is_lossless: + reasons.append(f"Lossless beats lossy ({fmt_codec(winner)} > {fmt_codec(runner_up)})") + if runner_up and winner.bitrate != runner_up.bitrate: + if winner.bitrate > runner_up.bitrate: + reasons.append(f"Higher bitrate: {winner.bitrate} vs {runner_up.bitrate} kbps") + else: + reasons.append(f"Bitrate tie broken by other properties ({winner.bitrate} vs {runner_up.bitrate} kbps)") + if runner_up and winner.sample_rate != runner_up.sample_rate: + if winner.sample_rate > runner_up.sample_rate: + reasons.append(f"Higher sample rate: {winner.sample_rate} Hz vs {runner_up.sample_rate} Hz") + else: + reasons.append("Sample rate tie handled by deterministic ordering") + if runner_up and winner.bit_depth != runner_up.bit_depth: + if winner.bit_depth > runner_up.bit_depth: + reasons.append(f"Higher bit depth: {winner.bit_depth}-bit vs {runner_up.bit_depth}-bit") + else: + reasons.append("Bit depth tie handled by deterministic ordering") + if runner_up and winner.channels != runner_up.channels: + if winner.channels > runner_up.channels: + reasons.append(f"More channels: {winner.channels} vs {runner_up.channels}") + else: + reasons.append("Channel count tie handled by deterministic ordering") return { "context": context, @@ -559,35 +730,82 @@ def _quality_rationale(winner: DuplicateTrack, runner_up: DuplicateTrack | None, "bitrate": winner.bitrate, "sample_rate": winner.sample_rate, "bit_depth": winner.bit_depth, + "channels": winner.channels, + "codec": winner.codec or "", + "container": winner.container or winner.ext.replace(".", "").upper(), "metadata_count": winner.metadata_count, "reasons": reasons or ["Deterministic ordering by quality"], + "runner_up": { + "bitrate": runner_up.bitrate if runner_up else None, + "sample_rate": runner_up.sample_rate if runner_up else None, + "bit_depth": runner_up.bit_depth if runner_up else None, + "channels": runner_up.channels if runner_up else None, + "codec": runner_up.codec if runner_up else "", + "container": runner_up.container if runner_up else "", + "path": runner_up.path if runner_up else None, + }, } -def _artwork_score(track: DuplicateTrack) -> tuple: - size_hint = 0 - has_art = False - if track.artwork: - has_art = True - size_hint = max(c.size for c in track.artwork if c.status == "ok") if any(c.status == "ok" for c in track.artwork) else 0 - return ( - 1 if has_art else 0, - size_hint, - track.bitrate, - track.sample_rate, - ) +def _artwork_blob_score(blob: ArtworkCandidate) -> tuple: + resolution = 0 + if blob.width and blob.height: + resolution = blob.width * blob.height + has_dims = 1 if blob.width and blob.height else 0 + return has_dims, resolution, blob.size -def _select_artwork_candidate(candidates: Sequence[DuplicateTrack]) -> tuple[DuplicateTrack | None, bool]: - usable = [c for c in candidates if c.artwork] - ranked = sorted(usable, key=lambda t: (_artwork_score(t), t.path.lower()), reverse=True) - if not ranked: - return None, False - if len(ranked) == 1: - return ranked[0], False - top_score = _artwork_score(ranked[0]) - ambiguous = sum(1 for c in ranked if _artwork_score(c) == top_score) > 1 - return ranked[0], ambiguous +def _best_artwork_blob(candidates: Sequence[ArtworkCandidate]) -> ArtworkCandidate | None: + usable = [c for c in candidates if c.status == "ok"] or list(candidates) + if not usable: + return None + return sorted(usable, key=lambda c: (_artwork_blob_score(c), c.hash), reverse=True)[0] + + +def _select_single_artwork_candidate( + candidates: Sequence[DuplicateTrack], contexts: Mapping[str, str] +) -> tuple[DuplicateTrack | None, ArtworkCandidate | None, bool]: + best_track: DuplicateTrack | None = None + best_blob: ArtworkCandidate | None = None + best_score: tuple | None = None + ambiguous = False + for track in candidates: + if contexts.get(track.path) != "single" or not track.artwork: + continue + blob = _best_artwork_blob(track.artwork) + if not blob: + continue + score = _artwork_blob_score(blob) + if best_score is None or score > best_score: + best_score = score + best_track = track + best_blob = blob + ambiguous = False + elif score == best_score: + ambiguous = True + return best_track, best_blob, ambiguous + + +def _select_overall_artwork_candidate(candidates: Sequence[DuplicateTrack]) -> tuple[DuplicateTrack | None, ArtworkCandidate | None, bool]: + best_track: DuplicateTrack | None = None + best_blob: ArtworkCandidate | None = None + best_score: tuple | None = None + ambiguous = False + for track in candidates: + if not track.artwork: + continue + blob = _best_artwork_blob(track.artwork) + if not blob: + continue + score = _artwork_blob_score(blob) + if best_score is None or score > best_score: + best_score = score + best_track = track + best_blob = blob + ambiguous = False + elif score == best_score: + ambiguous = True + return best_track, best_blob, ambiguous def _cluster_duplicates( @@ -711,16 +929,10 @@ def build_consolidation_plan( runner_up = quality_sorted[1] if len(quality_sorted) > 1 else None metadata_source = _select_metadata_source(cluster, contexts) - planned_tags: Dict[str, object] = _merge_tags(_blank_tags(), winner.current_tags) - tag_source: Optional[str] = None - if metadata_source: - planned_tags = _merge_tags(planned_tags, metadata_source.current_tags) - if contexts.get(metadata_source.path) == "album": - planned_tags.setdefault("album_type", "album") - elif contexts.get(metadata_source.path) == "single": - planned_tags.setdefault("album_type", "single") - tag_source = metadata_source.path - else: + planned_tags, tag_source, tag_source_reason, tag_source_evidence = _build_planned_tags( + winner, metadata_source, contexts + ) + if not metadata_source: review_flags.append(f"Missing metadata source for group containing {winner.path}.") meta_changes = _metadata_changes(winner, planned_tags) @@ -729,31 +941,102 @@ def build_consolidation_plan( group_review: List[str] = [] - single_candidates = [t for t in cluster if contexts.get(t.path) == "single"] - album_candidates = [t for t in cluster if contexts.get(t.path) == "album"] - art_candidates_order = single_candidates or album_candidates or cluster - distinct_hashes = {t.cover_hash for t in art_candidates_order if t.cover_hash} - tag_artwork_conflict = len(distinct_hashes) > 1 - best_art, ambiguous_art = _select_artwork_candidate(art_candidates_order) - ambiguous_art = ambiguous_art or tag_artwork_conflict + track_quality: Dict[str, Dict[str, object]] = { + t.path: { + "container": t.container or t.ext.replace(".", "").upper(), + "codec": t.codec, + "bitrate": t.bitrate, + "sample_rate": t.sample_rate, + "bit_depth": t.bit_depth, + "channels": t.channels, + "is_lossless": t.is_lossless, + } + for t in quality_sorted + } + + single_art_track, single_art_blob, ambiguous_single_art = _select_single_artwork_candidate(cluster, contexts) + overall_art_track, overall_art_blob, ambiguous_overall_art = _select_overall_artwork_candidate(cluster) artwork_actions: List[ArtworkDirective] = [] chosen_artwork_source: Dict[str, object] = {"path": None, "reason": ""} - artwork_status = "missing" - if best_art and best_art.artwork: - best_blob = next((c for c in best_art.artwork if c.status == "ok"), best_art.artwork[0]) + artwork_status = "unchanged" + artwork_evidence: List[str] = [] + + if single_art_blob: chosen_artwork_source = { - "path": best_art.path, - "hash": best_blob.hash, - "size": best_blob.size, - "reason": "best artwork candidate", + "path": single_art_track.path if single_art_track else None, + "hash": single_art_blob.hash, + "size": single_art_blob.size, + "width": single_art_blob.width, + "height": single_art_blob.height, + "context": "single", + "reason": "Best single artwork by resolution/size", } - artwork_status = "selected" - if not winner.cover_hash or winner.cover_hash != best_blob.hash: - reason = "Copy artwork from best candidate to winner" - artwork_actions.append(ArtworkDirective(source=best_art.path, target=winner.path, reason=reason)) + artwork_evidence.append( + f"Single artwork selected from {single_art_track.path if single_art_track else 'unknown'} " + f"({single_art_blob.width}x{single_art_blob.height}, {single_art_blob.size} bytes)" + ) + elif overall_art_blob: + chosen_artwork_source = { + "path": overall_art_track.path if overall_art_track else None, + "hash": overall_art_blob.hash, + "size": overall_art_blob.size, + "width": overall_art_blob.width, + "height": overall_art_blob.height, + "context": "fallback", + "reason": "No single artwork found; best available artwork", + } + artwork_evidence.append( + f"No single artwork found; using best available artwork from {overall_art_track.path if overall_art_track else 'unknown'} " + f"({overall_art_blob.width}x{overall_art_blob.height}, {overall_art_blob.size} bytes)" + ) + else: + chosen_artwork_source["reason"] = "No artwork candidates available" + artwork_status = "none found" + artwork_evidence.append("No readable artwork candidates were discovered.") + + if winner_context == "album" and single_art_blob and single_art_track: + if not winner.cover_hash or winner.cover_hash != single_art_blob.hash: + artwork_status = "apply" + artwork_actions.append( + ArtworkDirective( + source=single_art_track.path, + target=winner.path, + reason="Apply single artwork to album-context winner", + ) + ) + artwork_evidence.append("Album-context winner will receive single artwork to preserve single cover.") + else: + artwork_status = "unchanged" + chosen_artwork_source["reason"] = "Winner already has selected single artwork" + artwork_evidence.append("Winner already matches selected single artwork hash; no copy needed.") + elif winner_context == "album" and not single_art_blob and overall_art_blob and overall_art_track: + if not winner.cover_hash or winner.cover_hash != overall_art_blob.hash: + artwork_status = "apply" + artwork_actions.append( + ArtworkDirective( + source=overall_art_track.path, + target=winner.path, + reason="Fill missing album artwork from best available source", + ) + ) + artwork_evidence.append("No single artwork available; applying best available artwork to album winner.") + else: + artwork_evidence.append("Album winner artwork already matches best available source.") + else: + if winner_context == "single": + artwork_evidence.append("Single-context winner retains its own artwork; album art will not be pulled.") + chosen_artwork_source["reason"] = chosen_artwork_source.get("reason") or "Single winner keeps artwork" + elif not single_art_blob and not overall_art_blob: + group_review.append("No artwork available to apply.") + + cover_hashes = {t.cover_hash for t in cluster if t.cover_hash} + ambiguous_art = ambiguous_single_art or ambiguous_overall_art + if len(cover_hashes) > 1: + ambiguous_art = True + artwork_evidence.append("Conflicting embedded artwork hashes between candidates.") if ambiguous_art: review_flags.append(f"Artwork selection ambiguous for group {_stable_group_id([t.path for t in cluster])}.") - if not best_art: + if not (single_art_blob or overall_art_blob): missing_reason = "No artwork candidates available" if any(t.artwork_error for t in cluster): missing_reason = "Artwork present but unreadable" @@ -766,6 +1049,7 @@ def build_consolidation_plan( playlist_map = {loser: winner.path for loser in losers} group_id = _stable_group_id([t.path for t in cluster]) + group_confidence = "High (identical fingerprint cluster)" if ambiguous_art: group_review.append("Artwork selection requires review.") if not losers: @@ -815,6 +1099,11 @@ def build_consolidation_plan( context_evidence=context_evidence, tag_source=tag_source, placeholders_present=placeholders, + tag_source_reason=tag_source_reason, + tag_source_evidence=tag_source_evidence, + track_quality=track_quality, + group_confidence=group_confidence, + artwork_evidence=artwork_evidence, ) ) @@ -848,15 +1137,82 @@ def _render_metadata(changes: Dict[str, Dict[str, object]]) -> str: + "" ) - def _render_artwork(actions: List[ArtworkDirective]) -> str: - if not actions: - return "Winner artwork unchanged." + def _render_quality(group: GroupPlan) -> str: + winner = group.winner_path + q = group.track_quality.get(winner, {}) rows = [ - f"
  • Copy artwork from {_html_escape(act.source)} → {_html_escape(act.target)} " - f"({_html_escape(act.reason)})
  • " - for act in actions + ("Container / Codec", f"{_html_escape(q.get('container') or '—')} / {_html_escape(q.get('codec') or '—')}"), + ("Bitrate", f"{_html_escape(q.get('bitrate') or '0')} kbps"), + ("Sample Rate", f"{_html_escape(q.get('sample_rate') or '0')} Hz"), + ("Bit Depth", f"{_html_escape(q.get('bit_depth') or '0')} bit"), + ("Channels", _html_escape(q.get("channels") or "—")), ] - return "
      " + "".join(rows) + "
    " + table = ( + "" + "" + + "".join(f"" for label, value in rows) + + "" + ) + reasons = group.winner_quality.get("reasons") or [] + return ( + "
    " + f"

    Winner audio

    " + f"

    Context: {_html_escape(group.winner_quality.get('context', 'unknown'))}

    " + f"{table}" + "

    Why this is the winner

    " + "
      " + + "".join(f"
    • {_html_escape(r)}
    • " for r in reasons) + + "
    " + "
    " + ) + + def _render_losers(group: GroupPlan) -> str: + if not group.losers: + return "No losers to consolidate." + rows = [] + for loser in group.losers: + disposition = group.loser_disposition.get(loser, "quarantine") + rewrite = group.playlist_rewrites.get(loser, "n/a") + q = group.track_quality.get(loser, {}) + quality = f"{_html_escape(q.get('container') or '—')} / {_html_escape(q.get('codec') or '—')} ({_html_escape(q.get('bitrate') or '0')} kbps)" + rows.append( + f"{_html_escape(loser)}{quality}{_html_escape(disposition)}" + f"{_html_escape(rewrite)}" + ) + return ( + "" + "" + + "".join(rows) + + "" + ) + + def _render_artwork_section(group: GroupPlan) -> str: + source = group.chosen_artwork_source or {} + source_path = source.get("path") or "None" + source_hash = source.get("hash") or "unknown" + dims = "×".join(str(v) for v in (source.get("width"), source.get("height")) if v) if source.get("width") or source.get("height") else "unknown" + size = f"{source.get('size') or 0} bytes" + lines = [] + if group.artwork: + lines.append("

    Artwork will be applied

      ") + for act in group.artwork: + lines.append( + "
    • " + f"{_html_escape(act.source)} → {_html_escape(act.target)}" + f" ({_html_escape(source_hash)}, {dims}, {size}; {_html_escape(act.reason)})" + "
    • " + ) + lines.append("
    ") + else: + reason = source.get("reason") or group.artwork_status or "Winner artwork unchanged." + lines.append(f"

    Artwork unchanged — {_html_escape(reason)}

    ") + lines.append( + f"

    Artwork source: {_html_escape(source_path)} " + f"(hash={_html_escape(source_hash)}, size={_html_escape(size)}, dimensions={_html_escape(dims)})

    " + ) + if group.artwork_evidence: + lines.append("
      " + "".join(f"
    • {_html_escape(ev)}
    • " for ev in group.artwork_evidence) + "
    ") + return "".join(lines) review_required = plan.review_required_groups lines = [ @@ -871,6 +1227,9 @@ def _render_artwork(actions: List[ArtworkDirective]) -> str: ".pill{display:inline-block;padding:4px 8px;border-radius:12px;font-size:12px;}", ".ok{background:#e7f5e8;color:#266534;}", ".warn{background:#ffe8cc;color:#b05e00;}", + ".section{margin-top:8px;}", + ".losers td{font-size:0.9em;}", + "details{margin-top:6px;}", "", "

    Duplicate Consolidation Preview

    ", f"

    Generated at {plan.generated_at.isoformat()}

    ", @@ -890,34 +1249,26 @@ def _render_artwork(actions: List[ArtworkDirective]) -> str: ) lines.append("
    ".format(cls="review" if group.review_flags else "")) lines.append( - f"
    Group {group.group_id}
    {review_badge}
    " + f"
    Group {group.group_id}" + f"
    Confidence: {_html_escape(group.group_confidence)}
    {review_badge}
    " ) - lines.append(f"

    Winner: {_html_escape(group.winner_path)}

    ") - lines.append("
      ") - for loser in group.losers: - disposition = group.loser_disposition.get(loser, "quarantine") - lines.append( - f"
    • Loser: {_html_escape(loser)} → {_html_escape(disposition)} | " - f"Playlist rewrite → {_html_escape(group.playlist_rewrites.get(loser, 'n/a'))}
    • " - ) - lines.append("
    ") + lines.append(f"

    Winner: {_html_escape(group.winner_path)}

    ") + lines.append(_render_quality(group)) - lines.append("

    Quality rationale

    ") - reasons = group.winner_quality.get("reasons") or [] - lines.append( - "
      " - + "".join(f"
    • {_html_escape(r)}
    • " for r in reasons) - + "
    " - ) - lines.append("

    Metadata normalization

    ") + lines.append("

    Losers

    ") + lines.append(_render_losers(group)) + lines.append("
    ") + + lines.append("

    Metadata normalization

    ") lines.append(_render_metadata(group.metadata_changes)) + lines.append(f"

    Tag source: {_html_escape(group.tag_source or 'None')} — {_html_escape(group.tag_source_reason)}

    ") + if group.tag_source_evidence: + lines.append("
      " + "".join(f"
    • {_html_escape(ev)}
    • " for ev in group.tag_source_evidence) + "
    ") + lines.append("
    ") - lines.append("

    Artwork

    ") - lines.append(_render_artwork(group.artwork)) - art_source = group.chosen_artwork_source or {} - source_path = art_source.get("path") or "None" - source_reason = art_source.get("reason") or group.artwork_status - lines.append(f"

    Chosen artwork source: {_html_escape(source_path)} ({_html_escape(source_reason)})

    ") + lines.append("

    Artwork

    ") + lines.append(_render_artwork_section(group)) + lines.append("
    ") lines.append( f"

    Playlist impact: {group.playlist_impact.playlists} playlists, " @@ -944,6 +1295,12 @@ def _render_artwork(actions: List[ArtworkDirective]) -> str: for path in sorted(group.context_evidence.keys()) ) + "" + + "

    Artwork evidence

      " + + "".join(f"
    • {_html_escape(ev)}
    • " for ev in group.artwork_evidence) + + "
    " + + "

    Tag source selection

      " + + "".join(f"
    • {_html_escape(ev)}
    • " for ev in group.tag_source_evidence) + + "
    " + "" ) lines.append("
    ") diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index b205d15..2259adf 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -26,7 +26,10 @@ from indexer_control import IndexCancelled, cancel_event as global_cancel_event from utils.path_helpers import ensure_long_path -_MUTAGEN_AVAILABLE = importlib.util.find_spec("mutagen") is not None +try: + _MUTAGEN_AVAILABLE = importlib.util.find_spec("mutagen") is not None +except ValueError: # pragma: no cover - defensive: broken mutagen installs + _MUTAGEN_AVAILABLE = False if _MUTAGEN_AVAILABLE: from mutagen import File as MutagenFile # type: ignore else: # pragma: no cover - optional dependency diff --git a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py index 7a6131c..bb64d5e 100644 --- a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py +++ b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py @@ -766,9 +766,10 @@ def render_dry_run_html_from_plan( *, heading_text="Phase A – Exact Metadata", title_prefix="Music Index (Dry Run)", + coord: DryRunCoordinator | None = None, ): """Render a dry-run preview from an already computed move plan.""" - coord = DryRunCoordinator() + coord = coord or DryRunCoordinator() def _decision_annotation(decision: str, action: str | None = None): normalized_action = (action or "").lower() @@ -816,9 +817,13 @@ def build_exact_metadata_section() -> str: decision_map = {} decision_keys = set() decision_actions = {} + source_map = {} if plan_items: for item in plan_items: dst = item.get("destination") + src = item.get("source") + if dst and src: + source_map[dst] = src decision = (item.get("decision") or "").upper() annotation = _decision_annotation(decision, item.get("action")) if dst and annotation: @@ -858,7 +863,11 @@ def build_exact_metadata_section() -> str: label_text, css_class = annotation classes.append(css_class) label = f" {sanitize(label_text)}" - line = f"{indent}- {sanitize(fname)}{label}" + source_label = "" + src_path = source_map.get(node) + if src_path: + source_label = f" (from {sanitize(os.path.basename(src_path))})" + line = f"{indent}- {sanitize(fname)}{label}{source_label}" if tags_str: line += f" [{sanitize(tags_str)}]" lines.append(line) @@ -874,7 +883,7 @@ def build_exact_metadata_section() -> str: out.write( "\n\n\n \n " f"{title_prefix} – {sanitize(os.path.basename(root_path))}\n " - "\n\n\n" + "\n\n\n" ) out.write(html_body) if html_body and not html_body.endswith("\n"): @@ -913,7 +922,7 @@ def log_callback(msg): pass log_callback("5/6: Writing dry-run HTML…") - render_dry_run_html_from_plan(root_path, output_html_path, moves, tag_index) + render_dry_run_html_from_plan(root_path, output_html_path, moves, tag_index, coord=coord) log_callback(f"✓ Dry-run HTML written to: {output_html_path}") return {"moved": 0, "html": output_html_path, "dry_run": True} diff --git a/music_indexer_api.py b/music_indexer_api.py index 7a6131c..bb64d5e 100644 --- a/music_indexer_api.py +++ b/music_indexer_api.py @@ -766,9 +766,10 @@ def render_dry_run_html_from_plan( *, heading_text="Phase A – Exact Metadata", title_prefix="Music Index (Dry Run)", + coord: DryRunCoordinator | None = None, ): """Render a dry-run preview from an already computed move plan.""" - coord = DryRunCoordinator() + coord = coord or DryRunCoordinator() def _decision_annotation(decision: str, action: str | None = None): normalized_action = (action or "").lower() @@ -816,9 +817,13 @@ def build_exact_metadata_section() -> str: decision_map = {} decision_keys = set() decision_actions = {} + source_map = {} if plan_items: for item in plan_items: dst = item.get("destination") + src = item.get("source") + if dst and src: + source_map[dst] = src decision = (item.get("decision") or "").upper() annotation = _decision_annotation(decision, item.get("action")) if dst and annotation: @@ -858,7 +863,11 @@ def build_exact_metadata_section() -> str: label_text, css_class = annotation classes.append(css_class) label = f" {sanitize(label_text)}" - line = f"{indent}- {sanitize(fname)}{label}" + source_label = "" + src_path = source_map.get(node) + if src_path: + source_label = f" (from {sanitize(os.path.basename(src_path))})" + line = f"{indent}- {sanitize(fname)}{label}{source_label}" if tags_str: line += f" [{sanitize(tags_str)}]" lines.append(line) @@ -874,7 +883,7 @@ def build_exact_metadata_section() -> str: out.write( "\n\n\n \n " f"{title_prefix} – {sanitize(os.path.basename(root_path))}\n " - "\n\n\n" + "\n\n\n" ) out.write(html_body) if html_body and not html_body.endswith("\n"): @@ -913,7 +922,7 @@ def log_callback(msg): pass log_callback("5/6: Writing dry-run HTML…") - render_dry_run_html_from_plan(root_path, output_html_path, moves, tag_index) + render_dry_run_html_from_plan(root_path, output_html_path, moves, tag_index, coord=coord) log_callback(f"✓ Dry-run HTML written to: {output_html_path}") return {"moved": 0, "html": output_html_path, "dry_run": True} diff --git a/tests/test_duplicate_consolidation_guards.py b/tests/test_duplicate_consolidation_guards.py index 166ec0b..cf9c4ec 100644 --- a/tests/test_duplicate_consolidation_guards.py +++ b/tests/test_duplicate_consolidation_guards.py @@ -15,14 +15,27 @@ def _make_group(tmp_path, *, review_flags=None, disposition="quarantine"): winner_path=str(winner), losers=[str(loser)], planned_winner_tags={"title": "Winner"}, + winner_current_tags={"title": "Old"}, + current_tags={str(winner): {"title": "Old"}, str(loser): {"title": "Old"}}, metadata_changes={"title": {"from": "old", "to": "Winner"}}, winner_quality={"reasons": ["test"]}, artwork=[], + artwork_candidates=[], + chosen_artwork_source={}, + artwork_status="unchanged", loser_disposition={str(loser): disposition}, playlist_rewrites={str(loser): str(winner)}, playlist_impact=PlaylistImpact(playlists=1, entries=1), review_flags=review_flags or [], context_summary={"album": [], "single": [], "unknown": [str(winner)]}, + context_evidence={str(winner): [], str(loser): []}, + tag_source=str(winner), + placeholders_present=False, + tag_source_reason="winner", + tag_source_evidence=[], + track_quality={str(winner): {}, str(loser): {}}, + group_confidence="High", + artwork_evidence=[], ) diff --git a/tests/test_duplicate_consolidation_preview.py b/tests/test_duplicate_consolidation_preview.py index da27efe..771c4a0 100644 --- a/tests/test_duplicate_consolidation_preview.py +++ b/tests/test_duplicate_consolidation_preview.py @@ -1,4 +1,5 @@ import json +import hashlib import os from duplicate_consolidation import ( @@ -37,6 +38,18 @@ def _sample_tracks(root: str) -> list[dict[str, object]]: ] +def _artwork_payload(seed: str) -> dict[str, object]: + payload = seed.encode() + return { + "hash": hashlib.sha256(payload).hexdigest(), + "size": len(payload), + "width": 600, + "height": 600, + "status": "ok", + "bytes": payload, + } + + def test_plan_contains_metadata_and_quality_summary(tmp_path) -> None: plan = build_consolidation_plan(_sample_tracks(str(tmp_path))) assert plan.groups @@ -66,3 +79,108 @@ def test_preview_render_and_export(tmp_path) -> None: data = json.loads(json_path.read_text()) assert data["summary"]["groups"] == len(plan.groups) assert data["summary"]["review_required"] == plan.review_required_count + + +def test_album_winner_gets_single_artwork_applied(tmp_path) -> None: + album_path = os.path.join(tmp_path, "Album", "Song A.flac") + single_path = os.path.join(tmp_path, "Singles", "Song A.mp3") + tracks = [ + { + "path": album_path, + "fingerprint": "fp-art-apply", + "ext": ".flac", + "bitrate": 1000, + "sample_rate": 48000, + "bit_depth": 24, + "tags": {"title": "Song A", "album": "Album A", "track": 1, "album_type": "album"}, + }, + { + "path": single_path, + "fingerprint": "fp-art-apply", + "ext": ".mp3", + "bitrate": 320, + "sample_rate": 44100, + "bit_depth": 0, + "tags": {"title": "Song A", "album_type": "single"}, + "artwork": [_artwork_payload("single-art")], + }, + ] + + plan = build_consolidation_plan(tracks) + group = plan.groups[0] + + assert group.artwork_status == "apply" + assert group.artwork + assert group.artwork[0].source == single_path + assert group.artwork[0].target == album_path + assert group.chosen_artwork_source.get("context") == "single" + assert any("single artwork" in ev.lower() for ev in group.artwork_evidence) + + +def test_single_winner_keeps_artwork_even_with_album_cover(tmp_path) -> None: + single_path = os.path.join(tmp_path, "Singles", "Song B.flac") + album_path = os.path.join(tmp_path, "Album", "Song B.mp3") + tracks = [ + { + "path": single_path, + "fingerprint": "fp-single-winner", + "ext": ".flac", + "bitrate": 1100, + "sample_rate": 48000, + "bit_depth": 24, + "tags": {"title": "Song B", "album_type": "single"}, + "artwork": [_artwork_payload("single-winner")], + }, + { + "path": album_path, + "fingerprint": "fp-single-winner", + "ext": ".mp3", + "bitrate": 256, + "sample_rate": 44100, + "bit_depth": 0, + "tags": {"title": "Song B", "album": "Album B", "album_type": "album"}, + "artwork": [_artwork_payload("album-art")], + }, + ] + + plan = build_consolidation_plan(tracks) + group = plan.groups[0] + + assert group.winner_path == single_path + assert group.artwork_status != "apply" + assert not group.artwork # no cross-application when single wins + assert any("single-context winner" in ev.lower() for ev in group.artwork_evidence) + + +def test_metadata_normalization_prefers_album_source(tmp_path) -> None: + single_path = os.path.join(tmp_path, "Singles", "Song C.flac") + album_path = os.path.join(tmp_path, "Album", "Song C.mp3") + tracks = [ + { + "path": single_path, + "fingerprint": "fp-tag-source", + "ext": ".flac", + "bitrate": 900, + "sample_rate": 48000, + "bit_depth": 24, + "tags": {"title": "Song C", "album_type": "single"}, + }, + { + "path": album_path, + "fingerprint": "fp-tag-source", + "ext": ".mp3", + "bitrate": 320, + "sample_rate": 44100, + "bit_depth": 0, + "tags": {"title": "Song C", "album": "Album C", "albumartist": "Artist C", "track": 1, "year": 2020, "album_type": "album"}, + }, + ] + + plan = build_consolidation_plan(tracks) + group = plan.groups[0] + + assert group.tag_source == album_path + assert group.planned_winner_tags["album"] == "Album C" + assert group.planned_winner_tags["albumartist"] == "Artist C" + assert group.planned_winner_tags["year"] == 2020 + assert "album" in group.metadata_changes From 8c6431a91f6ac45d4491be9aff373968e5888023 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 21 Dec 2025 22:03:36 -0500 Subject: [PATCH 153/606] Harden consolidation execution safety and reporting --- duplicate_consolidation.py | 54 +++ duplicate_consolidation_executor.py | 338 ++++++++++++++++--- tests/test_duplicate_consolidation_guards.py | 33 ++ 3 files changed, 381 insertions(+), 44 deletions(-) diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index 03270ab..c64d7d8 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -320,6 +320,31 @@ def _placeholder_present(tags: Mapping[str, object]) -> bool: return False +def _capture_library_state(path: str) -> Dict[str, object]: + """Capture stable file attributes to detect changes between preview and execution.""" + + state: Dict[str, object] = {"exists": os.path.exists(path)} + if not state["exists"]: + return state + try: + stat = os.stat(path) + state["size"] = stat.st_size + state["mtime"] = int(stat.st_mtime) + except Exception as exc: # pragma: no cover - depends on filesystem permissions + state["stat_error"] = str(exc) + return state + + hasher = hashlib.sha256() + try: + with open(ensure_long_path(path), "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + hasher.update(chunk) + state["sha256"] = hasher.hexdigest() + except Exception as exc: # pragma: no cover - depends on filesystem permissions + state["hash_error"] = str(exc) + return state + + @dataclass class ArtworkCandidate: """Discovered artwork and its metadata.""" @@ -362,6 +387,7 @@ class DuplicateTrack: context_evidence: List[str] = field(default_factory=list) metadata_error: Optional[str] = None artwork_error: Optional[str] = None + library_state: Dict[str, object] = field(default_factory=dict) @property def cover_hash(self) -> str | None: @@ -454,6 +480,7 @@ class GroupPlan: track_quality: Dict[str, Dict[str, object]] group_confidence: str artwork_evidence: List[str] + library_state: Dict[str, Dict[str, object]] = field(default_factory=dict) def to_dict(self) -> Dict[str, object]: return { @@ -482,6 +509,7 @@ def to_dict(self) -> Dict[str, object]: "track_quality": {k: dict(v) for k, v in self.track_quality.items()}, "group_confidence": self.group_confidence, "artwork_evidence": list(self.artwork_evidence), + "library_state": {k: dict(v) for k, v in self.library_state.items()}, } @@ -493,6 +521,8 @@ class ConsolidationPlan: review_flags: List[str] = field(default_factory=list) generated_at: datetime.datetime = field(default_factory=lambda: datetime.datetime.now(datetime.timezone.utc)) placeholders_present: bool = False + source_snapshot: Dict[str, Dict[str, object]] = field(default_factory=dict) + plan_signature: str | None = None def to_dict(self) -> Dict[str, object]: return { @@ -500,6 +530,8 @@ def to_dict(self) -> Dict[str, object]: "review_flags": list(self.review_flags), "generated_at": self.generated_at.isoformat(), "placeholders_present": self.placeholders_present, + "source_snapshot": {k: dict(v) for k, v in self.source_snapshot.items()}, + "plan_signature": self.plan_signature, } @property @@ -512,12 +544,31 @@ def review_required_count(self) -> int: """Return total number of review-required groups.""" return len(self.review_required_groups) + def __post_init__(self) -> None: + if not self.source_snapshot: + snapshot: Dict[str, Dict[str, object]] = {} + for group in self.groups: + for path, state in group.library_state.items(): + snapshot[path] = dict(state) + self.source_snapshot = snapshot + if self.plan_signature is None: + canonical = json.dumps( + { + "generated_at": self.generated_at.isoformat(), + "groups": [g.to_dict() for g in self.groups], + "snapshot": {k: dict(v) for k, v in self.source_snapshot.items()}, + }, + sort_keys=True, + ) + self.plan_signature = hashlib.sha256(canonical.encode("utf-8")).hexdigest() + def _normalize_track(raw: Mapping[str, object]) -> DuplicateTrack: path = str(raw.get("path")) ext = os.path.splitext(path)[1].lower() or str(raw.get("ext", "")).lower() provided_tags = raw.get("tags") if isinstance(raw.get("tags"), Mapping) else {} current_tags, artwork, meta_err, art_err, audio_props = _read_tags_and_artwork(path, provided_tags) + library_state = _capture_library_state(path) provided_artwork: List[ArtworkCandidate] = [] for art in raw.get("artwork", []) if isinstance(raw.get("artwork"), list) else []: try: @@ -555,6 +606,7 @@ def _normalize_track(raw: Mapping[str, object]) -> DuplicateTrack: artwork=artwork, metadata_error=meta_err, artwork_error=art_err, + library_state=library_state, ) @@ -1066,6 +1118,7 @@ def build_consolidation_plan( group_review.append("Artwork extraction failed for at least one track.") playlist_impact = PlaylistImpact(playlists=len(losers), entries=len(losers)) + track_states = {t.path: dict(t.library_state) for t in cluster} artwork_candidates: List[ArtworkCandidate] = [] for t in cluster: @@ -1104,6 +1157,7 @@ def build_consolidation_plan( track_quality=track_quality, group_confidence=group_confidence, artwork_evidence=artwork_evidence, + library_state=track_states, ) ) diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index 2259adf..1689c43 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -13,6 +13,7 @@ from __future__ import annotations import datetime +import hashlib import importlib.util import json import os @@ -22,7 +23,7 @@ from dataclasses import dataclass, field from typing import Callable, Dict, Iterable, List, Mapping, MutableMapping, Sequence -from duplicate_consolidation import ArtworkDirective, ConsolidationPlan +from duplicate_consolidation import ArtworkDirective, ConsolidationPlan, _capture_library_state from indexer_control import IndexCancelled, cancel_event as global_cancel_event from utils.path_helpers import ensure_long_path @@ -247,10 +248,11 @@ def _extract_artwork_bytes(path: str) -> bytes | None: return None -def _apply_artwork(action: ArtworkDirective) -> bool: +def _apply_artwork(action: ArtworkDirective) -> tuple[bool, str]: payload = _extract_artwork_bytes(action.source) if payload is None: - return False + return False, "No artwork payload available to embed." + sidecar = f"{action.target}.artwork" directory = os.path.dirname(sidecar) os.makedirs(directory, exist_ok=True) @@ -266,22 +268,61 @@ def _apply_artwork(action: ArtworkDirective) -> bool: os.unlink(tmp_path) except OSError: pass + if MutagenFile is None: - return True - audio = MutagenFile(ensure_long_path(action.target)) + return True, "Mutagen unavailable; wrote artwork sidecar only." + + target_path = ensure_long_path(action.target) + audio = MutagenFile(target_path) if audio is None: - return True + return False, "Unsupported audio format for embedding." + + ext = os.path.splitext(action.source)[1].lower() + mime = "image/jpeg" + if ext == ".png": + mime = "image/png" + try: - pics = getattr(audio, "pictures", None) - if pics is not None: - pics.clear() - pics.append(payload) - elif hasattr(audio, "tags"): - audio.tags["APIC:Consolidation"] = payload - audio.save() - except Exception: - return False - return True + if hasattr(audio, "pictures"): + # FLAC/APE: replace front cover while preserving other pictures. + from mutagen.flac import Picture # type: ignore + + existing = [p for p in getattr(audio, "pictures") if getattr(p, "type", None) != 3] + pic = Picture() + pic.data = payload + pic.type = 3 + pic.mime = mime + pic.desc = "Front cover" + audio.clear_pictures() + for p in existing: + audio.add_picture(p) + audio.add_picture(pic) + audio.save() + return True, "Embedded FLAC/APE picture." + if audio.__class__.__module__.startswith("mutagen.mp4"): + from mutagen.mp4 import MP4, MP4Cover # type: ignore + + mp4 = audio if isinstance(audio, MP4) else MP4(target_path) + cover_format = MP4Cover.FORMAT_PNG if mime == "image/png" else MP4Cover.FORMAT_JPEG + mp4["covr"] = [MP4Cover(payload, imageformat=cover_format)] + mp4.save() + return True, "Embedded MP4 cover atom." + # Default to ID3/APIC handling + from mutagen.id3 import APIC, ID3, ID3NoHeaderError # type: ignore + + try: + tags = ID3(target_path) + except ID3NoHeaderError: + tags = ID3() + preserved = [frame for frame in tags.getall("APIC") if getattr(frame, "type", None) != 3] + tags.delall("APIC") + for frame in preserved: + tags.add(frame) + tags.add(APIC(encoding=3, mime=mime, type=3, desc="Front cover", data=payload)) + tags.save(target_path) + return True, "Embedded ID3 cover art." + except Exception as exc: + return False, f"Artwork embed failed: {exc}" def _apply_metadata(target: str, planned_tags: Mapping[str, object]) -> bool: @@ -324,6 +365,31 @@ def _quarantine_path(path: str, quarantine_root: str, library_root: str) -> str: return dest +def _compute_plan_signature(plan: ConsolidationPlan) -> str: + canonical = { + "generated_at": plan.generated_at.isoformat(), + "groups": [g.to_dict() for g in plan.groups], + "snapshot": {k: dict(v) for k, v in plan.source_snapshot.items()}, + } + text = json.dumps(canonical, sort_keys=True) + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def _detect_state_drift(expected: Mapping[str, object], actual: Mapping[str, object]) -> str | None: + if not expected: + return "Missing expected snapshot." + if not actual.get("exists"): + if expected.get("exists"): + return "File missing since preview." + return None + if expected.get("exists") is False and actual.get("exists"): + return "File appeared after preview." + for key in ("sha256", "size"): + if key in expected and key in actual and expected[key] != actual[key]: + return f"{key} mismatch (expected {expected.get(key)}, found {actual.get(key)})." + return None + + def execute_consolidation_plan(plan: ConsolidationPlan, config: ExecutionConfig) -> ExecutionResult: """Execute the provided consolidation plan with safety guarantees.""" @@ -346,12 +412,16 @@ def execute_consolidation_plan(plan: ConsolidationPlan, config: ExecutionConfig) loser_disposition: Dict[str, str] = {} artwork_actions: List[ArtworkDirective] = [] planned_tags: Dict[str, Dict[str, object]] = {} + path_to_group: Dict[str, str] = {} + group_lookup: Dict[str, object] = {g.group_id: g for g in plan.groups} for group in plan.groups: planned_tags[group.winner_path] = dict(group.planned_winner_tags) for loser in group.losers: loser_to_winner[loser] = group.winner_path loser_disposition[loser] = group.loser_disposition.get(loser, "quarantine") + path_to_group[loser] = group.group_id + path_to_group[group.winner_path] = group.group_id artwork_actions.extend(group.artwork) def _record(step: str, target: str, status: str, detail: str, **metadata: object) -> None: @@ -362,11 +432,49 @@ def _check_cancel(stage: str) -> None: _record(stage, "execution", "cancelled", "Execution cancelled by user request.") raise IndexCancelled() + def _find_loser_references() -> Dict[str, List[str]]: + refs: Dict[str, List[str]] = {} + for playlist in _iter_playlists(playlists_dir): + try: + with open(playlist, "r", encoding="utf-8") as handle: + lines = [ln.rstrip("\n") for ln in handle] + except FileNotFoundError: + continue + playlist_dir = os.path.dirname(playlist) + normalized = {_normalize_playlist_entry(ln, playlist_dir) for ln in lines if ln} + hits = sorted([loser for loser in loser_to_winner.keys() if loser in normalized]) + if hits: + refs[playlist] = hits + return refs + success = True + computed_signature = _compute_plan_signature(plan) try: _check_cancel("start") + if plan.plan_signature and plan.plan_signature != computed_signature: + success = False + _record( + "preflight", + "execution", + "blocked", + "Plan signature mismatch; rerun preview to regenerate the plan.", + expected_signature=plan.plan_signature, + computed_signature=computed_signature, + ) + raise RuntimeError("Execution blocked: plan signature mismatch.") + + if not plan.source_snapshot: + success = False + _record( + "preflight", + "execution", + "blocked", + "Plan missing source snapshot; rerun preview to capture library state.", + ) + raise RuntimeError("Execution blocked: missing source snapshot.") + if plan.review_required_count and not config.allow_review_required: success = False _record( @@ -391,6 +499,24 @@ def _check_cancel(stage: str) -> None: ) raise RuntimeError("Execution blocked: operation limit exceeded.") + drift: Dict[str, str] = {} + for path, expected in plan.source_snapshot.items(): + actual = _capture_library_state(path) + delta = _detect_state_drift(expected, actual) + if delta: + drift[path] = delta + if drift: + success = False + for path, reason in drift.items(): + _record( + "preflight", + path, + "failed", + f"Library changed since preview: {reason}", + group_id=path_to_group.get(path), + ) + raise RuntimeError("Execution blocked: library drift detected.") + has_deletions = any(disposition == "delete" for disposition in loser_disposition.values()) if has_deletions and not (config.allow_deletion or config.dry_run_execute): success = False @@ -405,6 +531,7 @@ def _check_cancel(stage: str) -> None: # Step 1: backup playlists that will change impacted: List[str] = [] + playlist_groups: Dict[str, set[str]] = {} for playlist in _iter_playlists(playlists_dir): try: with open(playlist, "r", encoding="utf-8") as handle: @@ -413,14 +540,25 @@ def _check_cancel(stage: str) -> None: continue playlist_dir = os.path.dirname(playlist) normalized = {_normalize_playlist_entry(ln, playlist_dir) for ln in lines if ln} - if normalized & loser_to_winner.keys(): + affected = normalized & loser_to_winner.keys() + if affected: impacted.append(playlist) + playlist_groups[playlist] = {path_to_group.get(path, "") for path in affected if path in path_to_group} + playlist_backups: Dict[str, str] = {} for playlist in impacted: _check_cancel("backup_playlists") try: backup_path = _backup_playlist(playlist, backups_dir) - _record("backup_playlists", playlist, "success", "Playlist backed up.", backup_path=backup_path) + playlist_backups[playlist] = backup_path + _record( + "backup_playlists", + playlist, + "success", + "Playlist backed up.", + backup_path=backup_path, + group_ids=sorted(g for g in playlist_groups.get(playlist, []) if g), + ) except Exception as exc: success = False _record("backup_playlists", playlist, "failed", f"Failed to backup playlist: {exc}") @@ -434,25 +572,38 @@ def _check_cancel(stage: str) -> None: playlist_results.append( PlaylistRewriteResult( playlist=playlist, - backup_path=_backup_playlist(playlist, backups_dir), + backup_path=playlist_backups.get(playlist, _backup_playlist(playlist, backups_dir)), replaced_entries=0, status="skipped", detail="No matching loser entries found.", ) ) - _record("rewrite_playlists", playlist, "skipped", "No entries required updates.") + _record( + "rewrite_playlists", + playlist, + "skipped", + "No entries required updates.", + group_ids=sorted(g for g in playlist_groups.get(playlist, []) if g), + ) continue try: _write_playlist_atomic(playlist, new_lines) playlist_results.append( PlaylistRewriteResult( playlist=playlist, - backup_path=os.path.join(backups_dir, os.path.relpath(playlist, playlists_dir)), + backup_path=playlist_backups.get(playlist, os.path.join(backups_dir, os.path.relpath(playlist, playlists_dir))), replaced_entries=replaced, status="success", ) ) - _record("rewrite_playlists", playlist, "success", "Playlist rewritten.", replaced=replaced) + _record( + "rewrite_playlists", + playlist, + "success", + "Playlist rewritten.", + replaced=replaced, + group_ids=sorted(g for g in playlist_groups.get(playlist, []) if g), + ) except Exception as exc: success = False _record("rewrite_playlists", playlist, "failed", f"Failed to rewrite playlist: {exc}") @@ -474,19 +625,47 @@ def _check_cancel(stage: str) -> None: "failed", "Playlist validation failed; restored backup.", missing=missing, + group_ids=sorted(g for g in playlist_groups.get(result.playlist, []) if g), ) raise RuntimeError(f"Playlist validation failed for {result.playlist}") - _record("validate_playlists", result.playlist, "success", "Playlist validated.") + _record( + "validate_playlists", + result.playlist, + "success", + "Playlist validated.", + group_ids=sorted(g for g in playlist_groups.get(result.playlist, []) if g), + ) + + lingering_refs = _find_loser_references() + if lingering_refs: + success = False + for playlist, losers in lingering_refs.items(): + _record( + "validate_playlists", + playlist, + "failed", + "Loser references remain after rewrite; aborting cleanup.", + losers=losers, + group_ids=sorted({path_to_group.get(l, "") for l in losers if l in path_to_group}), + ) + raise RuntimeError("Playlist rewrites incomplete; loser references remain.") # Step 4: apply artwork transfers if config.apply_artwork: for art in artwork_actions: _check_cancel("artwork") - ok = _apply_artwork(art) + ok, detail = _apply_artwork(art) status = "success" if ok else "failed" if not ok: success = False - _record("artwork", art.target, status, art.reason, source=art.source) + _record( + "artwork", + art.target, + status, + detail or art.reason, + source=art.source, + group_id=path_to_group.get(art.target), + ) if not ok: raise RuntimeError(f"Artwork copy failed for {art.target}") @@ -498,7 +677,7 @@ def _check_cancel(stage: str) -> None: status = "success" if ok else "failed" if not ok: success = False - _record("metadata", target, status, "Metadata normalized.") + _record("metadata", target, status, "Metadata normalized.", group_id=path_to_group.get(target)) if not ok: raise RuntimeError(f"Metadata normalization failed for {target}") @@ -513,6 +692,7 @@ def _check_cancel(stage: str) -> None: "skipped", "Dry-run execute enabled; loser not moved or deleted.", disposition=disposition, + group_id=path_to_group.get(loser), ) continue if disposition == "delete": @@ -520,10 +700,22 @@ def _check_cancel(stage: str) -> None: if os.path.exists(loser): os.remove(loser) quarantine_index[loser] = "deleted" - _record("loser_cleanup", loser, "success", "Loser deleted.") + _record( + "loser_cleanup", + loser, + "success", + "Loser deleted.", + group_id=path_to_group.get(loser), + ) except Exception as exc: success = False - _record("loser_cleanup", loser, "failed", f"Failed to delete loser: {exc}") + _record( + "loser_cleanup", + loser, + "failed", + f"Failed to delete loser: {exc}", + group_id=path_to_group.get(loser), + ) raise else: dest = _quarantine_path(loser, quarantine_dir, config.library_root) @@ -531,10 +723,23 @@ def _check_cancel(stage: str) -> None: if os.path.exists(loser): shutil.move(loser, dest) quarantine_index[loser] = dest - _record("loser_cleanup", loser, "success", "Loser quarantined.", quarantine_path=dest) + _record( + "loser_cleanup", + loser, + "success", + "Loser quarantined.", + quarantine_path=dest, + group_id=path_to_group.get(loser), + ) except Exception as exc: success = False - _record("loser_cleanup", loser, "failed", f"Failed to quarantine loser: {exc}") + _record( + "loser_cleanup", + loser, + "failed", + f"Failed to quarantine loser: {exc}", + group_id=path_to_group.get(loser), + ) raise except IndexCancelled: success = False @@ -550,6 +755,9 @@ def _check_cancel(stage: str) -> None: "success": success, "actions": [a.to_dict() for a in actions], "generated_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "plan_signature": plan.plan_signature or computed_signature, + "computed_signature": computed_signature, + "source_snapshot": {k: dict(v) for k, v in plan.source_snapshot.items()}, } playlist_payload = { "backups_dir": backups_dir, @@ -565,31 +773,73 @@ def _check_cancel(stage: str) -> None: pass try: + actions_by_group: Dict[str, List[ExecutionAction]] = {} + for action in actions: + meta = action.metadata or {} + group_ids: set[str] = set() + gid = meta.get("group_id") + if isinstance(gid, str) and gid: + group_ids.add(gid) + meta_groups = meta.get("group_ids") + if isinstance(meta_groups, (list, tuple, set)): + group_ids.update([g for g in meta_groups if isinstance(g, str) and g]) + if not group_ids: + actions_by_group.setdefault("__general__", []).append(action) + else: + for gid in group_ids: + actions_by_group.setdefault(gid, []).append(action) + html_lines = [ - "Duplicate Consolidation Execution", + "Duplicate Consolidation Execution", + "", "

    Execution Report

    ", f"

    Status: {'success' if success else 'failed'}

    ", - "
      ", + f"

      Plan signature: {plan.plan_signature or computed_signature}

      ", + f"

      Reports directory: {reports_dir}

      ", ] - for action in actions: - html_lines.append( - f"
    • {action.step} — {action.target}: {action.status} ({action.detail})" # noqa: E501 - "
    • " - ) - html_lines.append("
    ") + + for gid in sorted([g for g in actions_by_group.keys() if g != "__general__"]): + grp = group_lookup.get(gid) + winner_path = getattr(grp, "winner_path", "Unknown winner") + html_lines.append(f"

    Group {gid}

    ") + html_lines.append(f"

    Winner: {winner_path}

    ") + html_lines.append("
      ") + for act in actions_by_group.get(gid, []): + html_lines.append( + f"
    • {act.step} — {act.target}: " + f"{act.status} ({act.detail})
    • " + ) + html_lines.append("
    ") + + if actions_by_group.get("__general__"): + html_lines.append("

    General Actions

      ") + for act in actions_by_group["__general__"]: + html_lines.append( + f"
    • {act.step} — {act.target}: " + f"{act.status} ({act.detail})
    • " + ) + html_lines.append("
    ") + if playlist_results: - html_lines.append("

    Playlist Changes

      ") + html_lines.append("

      Playlist Changes

        ") for res in playlist_results: html_lines.append( - f"
      • {res.playlist} → {res.status} (replaced {res.replaced_entries}); " - f"backup: {res.backup_path}
      • " + f"
      • {res.playlist} → {res.status} " + f"(replaced {res.replaced_entries}); backup: {res.backup_path}
      • " ) - html_lines.append("
      ") + html_lines.append("
    ") if quarantine_index: - html_lines.append("

    Quarantined/Deleted

      ") + html_lines.append("

      Quarantined/Deleted

        ") for loser, dest in quarantine_index.items(): html_lines.append(f"
      • {loser} → {dest}
      • ") - html_lines.append("
      ") + html_lines.append("
    ") html_lines.append("") _atomic_write_text(html_report_path, "\n".join(html_lines)) except Exception: diff --git a/tests/test_duplicate_consolidation_guards.py b/tests/test_duplicate_consolidation_guards.py index cf9c4ec..d2b375b 100644 --- a/tests/test_duplicate_consolidation_guards.py +++ b/tests/test_duplicate_consolidation_guards.py @@ -1,3 +1,4 @@ +import hashlib import os import threading @@ -5,11 +6,21 @@ from duplicate_consolidation_executor import ExecutionConfig, execute_consolidation_plan +def _snapshot(path): + data = path.read_bytes() + stat = path.stat() + return {"exists": True, "size": stat.st_size, "mtime": int(stat.st_mtime), "sha256": hashlib.sha256(data).hexdigest()} + + def _make_group(tmp_path, *, review_flags=None, disposition="quarantine"): winner = tmp_path / "winner.flac" loser = tmp_path / "loser.mp3" winner.write_text("winner") loser.write_text("loser") + snapshot = { + str(winner): _snapshot(winner), + str(loser): _snapshot(loser), + } return GroupPlan( group_id="g1", winner_path=str(winner), @@ -36,6 +47,7 @@ def _make_group(tmp_path, *, review_flags=None, disposition="quarantine"): track_quality={str(winner): {}, str(loser): {}}, group_confidence="High", artwork_evidence=[], + library_state=snapshot, ) @@ -97,3 +109,24 @@ def test_dry_run_execute_skips_loser_cleanup(tmp_path): assert result.success is True assert result.quarantine_index[str(tmp_path / "loser.mp3")] == "retained" assert (tmp_path / "loser.mp3").exists() + + +def test_execution_blocks_library_drift(tmp_path): + plan = ConsolidationPlan(groups=[_make_group(tmp_path, review_flags=[])]) + loser_path = tmp_path / "loser.mp3" + loser_path.write_text("modified") + + result = execute_consolidation_plan(plan, _config(tmp_path, allow_review_required=True)) + + assert result.success is False + assert loser_path.exists() + + +def test_execution_blocks_signature_mismatch(tmp_path): + plan = ConsolidationPlan(groups=[_make_group(tmp_path, review_flags=[])]) + plan.plan_signature = "tampered" + + result = execute_consolidation_plan(plan, _config(tmp_path, allow_review_required=True)) + + assert result.success is False + assert (tmp_path / "loser.mp3").exists() From dc0bd637dbb91f2b325f7030e6ec185a6508e8ef Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 22 Dec 2025 06:43:16 -0500 Subject: [PATCH 154/606] Harden duplicate consolidation safety and dry run handling --- duplicate_consolidation.py | 30 ++- duplicate_consolidation_executor.py | 155 +++++++++----- .../test_duplicate_consolidation_hardening.py | 189 ++++++++++++++++++ 3 files changed, 326 insertions(+), 48 deletions(-) create mode 100644 tests/test_duplicate_consolidation_hardening.py diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index c64d7d8..9ec5cf3 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -542,7 +542,12 @@ def review_required_groups(self) -> List[GroupPlan]: @property def review_required_count(self) -> int: """Return total number of review-required groups.""" - return len(self.review_required_groups) + return len(self.review_required_groups) + (1 if self.review_flags else 0) + + @property + def requires_review(self) -> bool: + """Return whether any review flags (global or per-group) are present.""" + return bool(self.review_flags or self.review_required_groups) def __post_init__(self) -> None: if not self.source_snapshot: @@ -986,12 +991,20 @@ def build_consolidation_plan( ) if not metadata_source: review_flags.append(f"Missing metadata source for group containing {winner.path}.") + group_review = ["Metadata source missing; tags may be incomplete."] + else: + group_review: List[str] = [] meta_changes = _metadata_changes(winner, planned_tags) winner_context = contexts.get(winner.path, "unknown") winner_quality = _quality_rationale(winner, runner_up, winner_context) - group_review: List[str] = [] + distance_samples: List[float] = [] + missing_fingerprints = any(not t.fingerprint for t in cluster) + for idx, t in enumerate(cluster): + for other in cluster[idx + 1 :]: + distance_samples.append(fingerprint_distance(t.fingerprint, other.fingerprint)) + max_distance = max(distance_samples) if distance_samples else (1.0 if missing_fingerprints else 0.0) track_quality: Dict[str, Dict[str, object]] = { t.path: { @@ -1097,11 +1110,22 @@ def build_consolidation_plan( artwork_status = "none found" group_review.append(missing_reason) + required_tag_gaps = [key for key in ("artist", "title") if not planned_tags.get(key)] + if required_tag_gaps: + group_review.append(f"Missing critical tags: {', '.join(sorted(required_tag_gaps))}.") + + group_confidence = "High (identical fingerprint cluster)" + if missing_fingerprints: + group_confidence = "Low (missing fingerprints in cluster)" + group_review.append("One or more tracks missing fingerprints; requires review.") + if max_distance > 0: + group_confidence = f"Low (max fingerprint distance {max_distance:.3f})" + group_review.append(f"Audio match requires review (max distance {max_distance:.3f}).") + dispositions = {loser: "quarantine" for loser in losers} playlist_map = {loser: winner.path for loser in losers} group_id = _stable_group_id([t.path for t in cluster]) - group_confidence = "High (identical fingerprint cluster)" if ambiguous_art: group_review.append("Artwork selection requires review.") if not losers: diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index 1689c43..b67b61b 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -57,6 +57,7 @@ class ExecutionConfig: operation_limit: int | None = 500 confirm_operation_overage: bool = False allow_deletion: bool = False + confirm_deletion: bool = False dry_run_execute: bool = False @@ -89,6 +90,7 @@ class PlaylistRewriteResult: replaced_entries: int status: str detail: str = "" + original_playlist: str | None = None def to_dict(self) -> Dict[str, object]: return { @@ -97,6 +99,7 @@ def to_dict(self) -> Dict[str, object]: "replaced_entries": self.replaced_entries, "status": self.status, "detail": self.detail, + "original_playlist": self.original_playlist, } @@ -157,8 +160,8 @@ def _normalize_playlist_entry(entry: str, playlist_dir: str) -> str: return os.path.normpath(os.path.join(playlist_dir, entry)) -def _rewrite_playlist(path: str, mapping: Mapping[str, str]) -> tuple[int, List[str]]: - playlist_dir = os.path.dirname(path) +def _rewrite_playlist(path: str, mapping: Mapping[str, str], *, base_dir: str | None = None) -> tuple[int, List[str]]: + playlist_dir = base_dir or os.path.dirname(path) replaced = 0 try: with open(path, "r", encoding="utf-8") as handle: @@ -197,8 +200,8 @@ def _write_playlist_atomic(path: str, lines: Sequence[str]) -> None: pass -def _validate_playlist(path: str) -> List[str]: - playlist_dir = os.path.dirname(path) +def _validate_playlist(path: str, *, base_dir: str | None = None) -> List[str]: + playlist_dir = base_dir or os.path.dirname(path) try: with open(path, "r", encoding="utf-8") as handle: lines = [ln.rstrip("\n") for ln in handle] @@ -402,6 +405,7 @@ def execute_consolidation_plan(plan: ConsolidationPlan, config: ExecutionConfig) run_root = _timestamped_dir(config.reports_dir) reports_dir = os.path.join(run_root, "reports") backups_dir = os.path.join(run_root, "playlist_backups") + validation_playlists_dir = os.path.join(run_root, "playlist_validation") os.makedirs(reports_dir, exist_ok=True) os.makedirs(backups_dir, exist_ok=True) @@ -414,6 +418,8 @@ def execute_consolidation_plan(plan: ConsolidationPlan, config: ExecutionConfig) planned_tags: Dict[str, Dict[str, object]] = {} path_to_group: Dict[str, str] = {} group_lookup: Dict[str, object] = {g.group_id: g for g in plan.groups} + playlist_targets: Dict[str, str] = {} + playlist_base_dirs: Dict[str, str] = {} for group in plan.groups: planned_tags[group.winner_path] = dict(group.planned_winner_tags) @@ -432,15 +438,18 @@ def _check_cancel(stage: str) -> None: _record(stage, "execution", "cancelled", "Execution cancelled by user request.") raise IndexCancelled() - def _find_loser_references() -> Dict[str, List[str]]: + def _find_loser_references( + working_playlists: Mapping[str, str], + playlist_base_dirs: Mapping[str, str] | None, + ) -> Dict[str, List[str]]: refs: Dict[str, List[str]] = {} - for playlist in _iter_playlists(playlists_dir): + for playlist, working_path in working_playlists.items(): try: - with open(playlist, "r", encoding="utf-8") as handle: + with open(working_path, "r", encoding="utf-8") as handle: lines = [ln.rstrip("\n") for ln in handle] except FileNotFoundError: continue - playlist_dir = os.path.dirname(playlist) + playlist_dir = (playlist_base_dirs or {}).get(playlist) or os.path.dirname(working_path) normalized = {_normalize_playlist_entry(ln, playlist_dir) for ln in lines if ln} hits = sorted([loser for loser in loser_to_winner.keys() if loser in normalized]) if hits: @@ -475,7 +484,7 @@ def _find_loser_references() -> Dict[str, List[str]]: ) raise RuntimeError("Execution blocked: missing source snapshot.") - if plan.review_required_count and not config.allow_review_required: + if plan.requires_review and not config.allow_review_required: success = False _record( "preflight", @@ -518,16 +527,17 @@ def _find_loser_references() -> Dict[str, List[str]]: raise RuntimeError("Execution blocked: library drift detected.") has_deletions = any(disposition == "delete" for disposition in loser_disposition.values()) - if has_deletions and not (config.allow_deletion or config.dry_run_execute): - success = False - _record( - "preflight", - "execution", - "blocked", - "Deletion requested but confirmation not provided.", - deletions_requested=sum(1 for d in loser_disposition.values() if d == "delete"), - ) - raise RuntimeError("Execution blocked: deletions require confirmation.") + if has_deletions and not config.dry_run_execute: + if not (config.allow_deletion and config.confirm_deletion): + success = False + _record( + "preflight", + "execution", + "blocked", + "Deletion requested but explicit confirmation was not provided.", + deletions_requested=sum(1 for d in loser_disposition.values() if d == "delete"), + ) + raise RuntimeError("Execution blocked: deletions require confirmation.") # Step 1: backup playlists that will change impacted: List[str] = [] @@ -549,16 +559,34 @@ def _find_loser_references() -> Dict[str, List[str]]: for playlist in impacted: _check_cancel("backup_playlists") try: - backup_path = _backup_playlist(playlist, backups_dir) - playlist_backups[playlist] = backup_path - _record( - "backup_playlists", - playlist, - "success", - "Playlist backed up.", - backup_path=backup_path, - group_ids=sorted(g for g in playlist_groups.get(playlist, []) if g), - ) + if config.dry_run_execute: + working_copy = os.path.join(validation_playlists_dir, os.path.relpath(playlist, playlists_dir)) + os.makedirs(os.path.dirname(working_copy), exist_ok=True) + shutil.copy2(playlist, working_copy) + playlist_backups[playlist] = working_copy + playlist_targets[playlist] = working_copy + playlist_base_dirs[playlist] = os.path.dirname(playlist) + _record( + "backup_playlists", + playlist, + "success", + "Playlist copied for dry-run validation.", + backup_path=working_copy, + group_ids=sorted(g for g in playlist_groups.get(playlist, []) if g), + ) + else: + backup_path = _backup_playlist(playlist, backups_dir) + playlist_backups[playlist] = backup_path + playlist_targets[playlist] = playlist + playlist_base_dirs[playlist] = os.path.dirname(playlist) + _record( + "backup_playlists", + playlist, + "success", + "Playlist backed up.", + backup_path=backup_path, + group_ids=sorted(g for g in playlist_groups.get(playlist, []) if g), + ) except Exception as exc: success = False _record("backup_playlists", playlist, "failed", f"Failed to backup playlist: {exc}") @@ -567,52 +595,65 @@ def _find_loser_references() -> Dict[str, List[str]]: # Step 2: rewrite playlists for playlist in impacted: _check_cancel("rewrite_playlists") - replaced, new_lines = _rewrite_playlist(playlist, loser_to_winner) + target_playlist = playlist_targets.get(playlist, playlist) + base_dir = playlist_base_dirs.get(playlist, os.path.dirname(target_playlist)) + replaced, new_lines = _rewrite_playlist(target_playlist, loser_to_winner, base_dir=base_dir) if replaced == 0: playlist_results.append( PlaylistRewriteResult( - playlist=playlist, + playlist=target_playlist, backup_path=playlist_backups.get(playlist, _backup_playlist(playlist, backups_dir)), replaced_entries=0, status="skipped", - detail="No matching loser entries found.", + detail="No matching loser entries found." + (" (dry-run copy)" if config.dry_run_execute else ""), + original_playlist=playlist if target_playlist != playlist else None, ) ) _record( "rewrite_playlists", - playlist, + target_playlist, "skipped", "No entries required updates.", group_ids=sorted(g for g in playlist_groups.get(playlist, []) if g), + original_playlist=playlist if target_playlist != playlist else None, ) continue try: - _write_playlist_atomic(playlist, new_lines) + _write_playlist_atomic(target_playlist, new_lines) playlist_results.append( PlaylistRewriteResult( - playlist=playlist, - backup_path=playlist_backups.get(playlist, os.path.join(backups_dir, os.path.relpath(playlist, playlists_dir))), + playlist=target_playlist, + backup_path=playlist_backups.get( + playlist, + os.path.join( + backups_dir, + os.path.relpath(playlist, playlists_dir), + ), + ), replaced_entries=replaced, status="success", + original_playlist=playlist if target_playlist != playlist else None, ) ) _record( "rewrite_playlists", - playlist, + target_playlist, "success", "Playlist rewritten.", replaced=replaced, group_ids=sorted(g for g in playlist_groups.get(playlist, []) if g), + original_playlist=playlist if target_playlist != playlist else None, ) except Exception as exc: success = False - _record("rewrite_playlists", playlist, "failed", f"Failed to rewrite playlist: {exc}") + _record("rewrite_playlists", target_playlist, "failed", f"Failed to rewrite playlist: {exc}") raise # Step 3: validate rewritten playlists for result in playlist_results: _check_cancel("validate_playlists") - missing = _validate_playlist(result.playlist) + base_dir = playlist_base_dirs.get(result.original_playlist or result.playlist, os.path.dirname(result.playlist)) + missing = _validate_playlist(result.playlist, base_dir=base_dir) if missing: success = False try: @@ -625,7 +666,8 @@ def _find_loser_references() -> Dict[str, List[str]]: "failed", "Playlist validation failed; restored backup.", missing=missing, - group_ids=sorted(g for g in playlist_groups.get(result.playlist, []) if g), + group_ids=sorted(g for g in playlist_groups.get(result.original_playlist or result.playlist, []) if g), + original_playlist=result.original_playlist, ) raise RuntimeError(f"Playlist validation failed for {result.playlist}") _record( @@ -633,10 +675,11 @@ def _find_loser_references() -> Dict[str, List[str]]: result.playlist, "success", "Playlist validated.", - group_ids=sorted(g for g in playlist_groups.get(result.playlist, []) if g), + group_ids=sorted(g for g in playlist_groups.get(result.original_playlist or result.playlist, []) if g), + original_playlist=result.original_playlist, ) - lingering_refs = _find_loser_references() + lingering_refs = _find_loser_references(playlist_targets or {}, playlist_base_dirs) if lingering_refs: success = False for playlist, losers in lingering_refs.items(): @@ -651,7 +694,17 @@ def _find_loser_references() -> Dict[str, List[str]]: raise RuntimeError("Playlist rewrites incomplete; loser references remain.") # Step 4: apply artwork transfers - if config.apply_artwork: + if config.dry_run_execute and config.apply_artwork: + for art in artwork_actions: + _record( + "artwork", + art.target, + "skipped", + "Dry-run execute enabled; artwork not applied.", + source=art.source, + group_id=path_to_group.get(art.target), + ) + elif config.apply_artwork: for art in artwork_actions: _check_cancel("artwork") ok, detail = _apply_artwork(art) @@ -670,7 +723,16 @@ def _find_loser_references() -> Dict[str, List[str]]: raise RuntimeError(f"Artwork copy failed for {art.target}") # Step 5: apply metadata normalization - if config.apply_metadata: + if config.dry_run_execute and config.apply_metadata: + for target, _tags in planned_tags.items(): + _record( + "metadata", + target, + "skipped", + "Dry-run execute enabled; metadata not written.", + group_id=path_to_group.get(target), + ) + elif config.apply_metadata: for target, tags in planned_tags.items(): _check_cancel("metadata") ok = _apply_metadata(target, tags) @@ -830,8 +892,11 @@ def _find_loser_references() -> Dict[str, List[str]]: if playlist_results: html_lines.append("

    Playlist Changes

      ") for res in playlist_results: + label = res.playlist + if res.original_playlist: + label = f"{res.original_playlist} (validated copy: {res.playlist})" html_lines.append( - f"
    • {res.playlist} → {res.status} " + f"
    • {label} → {res.status} " f"(replaced {res.replaced_entries}); backup: {res.backup_path}
    • " ) html_lines.append("
    ") diff --git a/tests/test_duplicate_consolidation_hardening.py b/tests/test_duplicate_consolidation_hardening.py new file mode 100644 index 0000000..bfa4438 --- /dev/null +++ b/tests/test_duplicate_consolidation_hardening.py @@ -0,0 +1,189 @@ +import hashlib +import os +import threading + +from duplicate_consolidation import ( + ArtworkDirective, + ConsolidationPlan, + DuplicateTrack, + PlaylistImpact, + GroupPlan, + _extract_artwork_from_audio, + _metadata_changes, + build_consolidation_plan, +) +from duplicate_consolidation_executor import ExecutionConfig, _apply_artwork, execute_consolidation_plan + + +def _snapshot(path): + data = path.read_bytes() + stat = path.stat() + return {"exists": True, "size": stat.st_size, "mtime": int(stat.st_mtime), "sha256": hashlib.sha256(data).hexdigest()} + + +def _config(tmp_path, **overrides): + defaults = dict( + library_root=str(tmp_path), + reports_dir=str(tmp_path / "reports"), + playlists_dir=str(tmp_path / "Playlists"), + quarantine_dir=str(tmp_path / "Quarantine"), + cancel_event=threading.Event(), + log_callback=lambda _m: None, + ) + defaults.update(overrides) + return ExecutionConfig(**defaults) + + +def test_artwork_extraction_and_embedding(monkeypatch, tmp_path): + class DummyPic: + def __init__(self, payload: bytes): + self.data = payload + self.width = 320 + self.height = 240 + + class DummyAudio: + def __init__(self, payload: bytes): + self.pictures = [DummyPic(payload)] + self.tags = {} + + payload = b"art-bytes" + art, error = _extract_artwork_from_audio(DummyAudio(payload), "song.flac") + assert not error + assert len(art) == 1 + assert art[0].width == 320 + assert art[0].size == len(payload) + + source = tmp_path / "source.flac" + target = tmp_path / "target.flac" + source.write_text("audio") + target.write_text("audio") + (source.with_suffix(source.suffix + ".artwork")).write_bytes(payload) + + monkeypatch.setattr("duplicate_consolidation_executor.MutagenFile", None) + ok, detail = _apply_artwork(ArtworkDirective(source=str(source), target=str(target), reason="test copy")) + assert ok + assert "Mutagen unavailable" in detail + assert (tmp_path / "target.flac.artwork").read_bytes() == payload + + +def test_metadata_changes_diff_generation(): + winner = DuplicateTrack( + path="track.flac", + fingerprint="fp", + ext=".flac", + bitrate=256, + sample_rate=44100, + bit_depth=16, + channels=2, + tags={}, + current_tags={"artist": "Old Artist", "title": "Keep", "album": None}, + ) + planned = {"artist": "New Artist", "title": "Keep", "album": "Album Name"} + changes = _metadata_changes(winner, planned) + + assert changes["artist"]["from"] == "Old Artist" + assert changes["artist"]["to"] == "New Artist" + assert "title" not in changes # unchanged + assert changes["album"]["from"] is None + assert changes["album"]["to"] == "Album Name" + + +def test_playlist_rewrite_validation_dry_run(tmp_path): + playlists_dir = tmp_path / "Playlists" + playlists_dir.mkdir() + winner = tmp_path / "winner.flac" + loser = tmp_path / "loser.mp3" + winner.write_text("winner") + loser.write_text("loser") + + playlist = playlists_dir / "mix.m3u" + playlist.write_text(os.path.relpath(str(loser), playlists_dir) + "\n") + + snapshot = {str(winner): _snapshot(winner), str(loser): _snapshot(loser)} + plan = ConsolidationPlan( + groups=[ + GroupPlan( + group_id="g-dryrun", + winner_path=str(winner), + losers=[str(loser)], + planned_winner_tags={"title": "Winner"}, + winner_current_tags={"title": "Old"}, + current_tags={str(winner): {"title": "Old"}, str(loser): {"title": "Old"}}, + metadata_changes={"title": {"from": "Old", "to": "Winner"}}, + winner_quality={"reasons": ["test"]}, + artwork=[], + artwork_candidates=[], + chosen_artwork_source={}, + artwork_status="unchanged", + loser_disposition={str(loser): "quarantine"}, + playlist_rewrites={str(loser): str(winner)}, + playlist_impact=PlaylistImpact(playlists=1, entries=1), + review_flags=[], + context_summary={"album": [], "single": [], "unknown": [str(winner)]}, + context_evidence={str(winner): [], str(loser): []}, + tag_source=str(winner), + placeholders_present=False, + tag_source_reason="winner", + tag_source_evidence=[], + track_quality={str(winner): {}, str(loser): {}}, + group_confidence="High", + artwork_evidence=[], + library_state=snapshot, + ) + ] + ) + + cfg = _config( + tmp_path, + allow_review_required=True, + dry_run_execute=True, + ) + + result = execute_consolidation_plan(plan, cfg) + assert result.success is True + assert playlist.read_text().strip() == os.path.relpath(str(loser), playlists_dir) + assert result.playlist_reports + rewritten = result.playlist_reports[0].playlist + assert rewritten != str(playlist) + rewritten_text = os.path.relpath(str(winner), playlists_dir) + with open(rewritten, encoding="utf-8") as handle: + rewritten_lines = [ln.strip() for ln in handle] + assert rewritten_text in rewritten_lines + assert (tmp_path / "loser.mp3").exists() + + +def test_missing_tags_and_placeholders_require_review(tmp_path): + winner = tmp_path / "winner.flac" + loser = tmp_path / "loser.flac" + winner.write_text("winner") + loser.write_text("loser") + + tracks = [ + {"path": str(winner), "fingerprint": "fp-tags", "ext": ".flac", "tags": {"title": "Demo Track"}}, + {"path": str(loser), "fingerprint": "fp-tags", "ext": ".flac", "tags": {"title": "Demo Track"}}, + ] + plan = build_consolidation_plan(tracks) + + assert plan.requires_review + assert plan.review_required_count > 0 + assert any("placeholder" in flag.lower() or "missing critical" in flag.lower() for flag in plan.groups[0].review_flags) + + cfg = _config(tmp_path) + result = execute_consolidation_plan(plan, cfg) + assert result.success is False + assert (tmp_path / "loser.flac").exists() + + +def test_truncation_sets_review_required_count(tmp_path): + tracks = [ + {"path": str(tmp_path / f"track-{idx}.flac"), "fingerprint": f"fp-{idx}", "ext": ".flac"} + for idx in range(10) + ] + for track in tracks: + open(track["path"], "w").close() + + plan = build_consolidation_plan(tracks, max_candidates=5) + + assert plan.review_flags + assert plan.requires_review + assert plan.review_required_count >= 1 From 92084100513ddab8ab77317b229ff075d1743792 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 22 Dec 2025 06:49:33 -0500 Subject: [PATCH 155/606] Normalize tag values for consolidation plan --- duplicate_consolidation.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index 9ec5cf3..35e2f51 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -87,7 +87,22 @@ def _blank_tags() -> Dict[str, object]: def _first_value(value: object) -> object: if isinstance(value, list): return _first_value(value[0]) if value else None - return value + if isinstance(value, (str, int, float, bool)) or value is None: + return value + if isinstance(value, tuple): + return _first_value(value[0]) if value else None + if hasattr(value, "text"): + try: + text_value = value.text + if isinstance(text_value, (list, tuple)): + return _first_value(text_value[0]) if text_value else None + return str(text_value) if text_value is not None else None + except Exception: + pass + try: + return str(value) + except Exception: + return None def _parse_int(value: object) -> int | None: From a39f0888352069867734a171ceb08f63b62aa6c8 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 22 Dec 2025 19:03:26 -0500 Subject: [PATCH 156/606] Improve duplicate grouping metadata and coarse gating --- duplicate_consolidation.py | 148 ++++++++++++++++-- .../test_duplicate_consolidation_hardening.py | 62 ++++++++ 2 files changed, 200 insertions(+), 10 deletions(-) diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index 35e2f51..af0cb66 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -15,10 +15,12 @@ import io import json import os +import re import threading import time from dataclasses import dataclass, field from typing import Callable, Dict, Iterable, List, Mapping, MutableMapping, Optional, Sequence +from collections import defaultdict from utils.path_helpers import ensure_long_path try: @@ -37,7 +39,8 @@ else: # pragma: no cover - optional dependency Image = None # type: ignore -from near_duplicate_detector import fingerprint_distance +from music_indexer_api import extract_primary_and_collabs +from near_duplicate_detector import fingerprint_distance, _parse_fp as _parse_fingerprint LOSSLESS_EXTS = {".flac", ".wav", ".alac", ".ape", ".aiff", ".aif"} DEFAULT_DISTANCE_THRESHOLD = 0.0 @@ -70,6 +73,20 @@ "genre", "compilation", ] +TITLE_JUNK_TOKENS = { + "remaster", + "remastered", + "explicit", + "clean", + "mono", + "stereo", + "version", + "deluxe", + "bonus", +} +COARSE_FP_BAND_SIZE = 8 +COARSE_FP_BANDS = 6 +COARSE_FP_QUANTIZATION = 4 def _now() -> float: @@ -113,6 +130,103 @@ def _parse_int(value: object) -> int | None: return None +def _normalized_tokens(text: str) -> List[str]: + lowered = (text or "").lower() + cleaned = re.sub(r"[\\[\\]{}()]+", " ", lowered) + tokens = [tok for tok in re.split(r"[^a-z0-9]+", cleaned) if tok] + return tokens + + +def _normalize_primary_artist(tags: Mapping[str, object], path: str) -> str: + raw_artist = str(tags.get("artist") or tags.get("albumartist") or "").strip() + lowered = raw_artist.lower() + placeholders = {"various", "various artists", "va", "unknown"} + if lowered in placeholders: + raw_artist = "" + if not raw_artist: + stem = os.path.splitext(os.path.basename(path))[0] + if " - " in stem: + candidate = stem.split(" - ", 1)[0].strip() + if candidate.lower() not in placeholders: + raw_artist = candidate + primary, _ = extract_primary_and_collabs(raw_artist) if raw_artist else ("", []) + tokens = _normalized_tokens(primary) + return " ".join(tokens or ["unknown"]) + + +def _normalize_title(tags: Mapping[str, object], path: str) -> str: + raw_title = str(tags.get("title") or "").strip() + if not raw_title: + raw_title = os.path.splitext(os.path.basename(path))[0] + tokens = _normalized_tokens(raw_title) + filtered = [tok for tok in tokens if tok not in TITLE_JUNK_TOKENS] + filtered = [tok for tok in filtered if not (tok.isdigit() and len(tok) == 4)] + return " ".join(filtered or tokens or ["unknown"]) + + +def _metadata_bucket_key(track: DuplicateTrack) -> tuple[str, str]: + tags = track.current_tags if isinstance(track.current_tags, Mapping) else track.tags + tags = tags or {} + return _normalize_primary_artist(tags, track.path), _normalize_title(tags, track.path) + + +def _build_metadata_buckets(tracks: Sequence[DuplicateTrack]) -> Dict[tuple[str, str], List[DuplicateTrack]]: + title_to_artists: Dict[str, set[str]] = defaultdict(set) + normalized: List[tuple[DuplicateTrack, tuple[str, str]]] = [] + for track in tracks: + key = _metadata_bucket_key(track) + normalized.append((track, key)) + artist, title = key + if artist and artist != "unknown": + title_to_artists[title].add(artist) + buckets: Dict[tuple[str, str], List[DuplicateTrack]] = defaultdict(list) + for track, (artist, title) in normalized: + artist_key = artist + known_artists = title_to_artists.get(title, set()) + if artist_key == "unknown" and len(known_artists) == 1: + artist_key = next(iter(known_artists)) + buckets[(artist_key, title)].append(track) + return buckets + + +def _fingerprint_to_ints(fp: str | None, *, limit: int = COARSE_FP_BAND_SIZE * COARSE_FP_BANDS) -> List[int]: + if not fp: + return [] + parsed = _parse_fingerprint(fp) if "_parse_fingerprint" in globals() else None + ints: List[int] = [] + if parsed: + kind, payload = parsed + if kind == "ints": + ints = list(payload) + elif kind == "bytes": + ints = list(payload) + if not ints: + try: + ints = [int(tok) for tok in fp.replace(",", " ").split() if tok] + except Exception: + ints = [ord(ch) for ch in fp] + return ints[:limit] + + +def _coarse_fingerprint_keys(fp: str | None) -> List[str]: + if not fp: + return [] + ints = _fingerprint_to_ints(fp) + if not ints: + return [] + quantized = [val // COARSE_FP_QUANTIZATION for val in ints] + keys: List[str] = [] + for idx in range(0, len(quantized), COARSE_FP_BAND_SIZE): + band = quantized[idx : idx + COARSE_FP_BAND_SIZE] + if not band: + break + digest = hashlib.blake2s(",".join(map(str, band)).encode("utf-8"), digest_size=4).hexdigest() + keys.append(f"{idx // COARSE_FP_BAND_SIZE}:{digest}") + if not keys: + keys.append(f"band0:{hashlib.blake2s(fp.encode('utf-8'), digest_size=4).hexdigest()}") + return keys + + def _extract_image_dimensions(payload: bytes) -> tuple[Optional[int], Optional[int]]: if not payload or not Image: return None, None @@ -891,22 +1005,28 @@ def _cluster_duplicates( start_time: float, review_flags: List[str], ) -> List[List[DuplicateTrack]]: - buckets: Dict[str, List[DuplicateTrack]] = {} - for track in tracks: - if not track.fingerprint: - continue - key = str(track.fingerprint) if threshold <= 0 else str(track.fingerprint)[:20] - buckets.setdefault(key, []).append(track) - + buckets = _build_metadata_buckets(tracks) + bucket_items = sorted(buckets.items(), key=lambda x: x[0]) clusters: List[List[DuplicateTrack]] = [] comparisons = 0 processed = 0 - bucket_items = sorted(buckets.items(), key=lambda x: x[0]) for _, bucket_tracks in bucket_items: if cancel_event.is_set() or (timeout_sec and (_now() - start_time) > timeout_sec): review_flags.append("Consolidation planning cancelled or timed out during grouping.") break + bucket_tracks = [t for t in bucket_tracks if t.fingerprint] + if not bucket_tracks: + continue bucket_tracks = sorted(bucket_tracks, key=lambda t: t.path.lower()) + path_to_track = {t.path: t for t in bucket_tracks} + path_order = {t.path: idx for idx, t in enumerate(bucket_tracks)} + + track_keys: Dict[str, List[str]] = {t.path: _coarse_fingerprint_keys(t.fingerprint) for t in bucket_tracks} + coarse_index: Dict[str, set[str]] = defaultdict(set) + for path, keys in track_keys.items(): + for key in keys: + coarse_index[key].add(path) + used: set[str] = set() for idx, track in enumerate(bucket_tracks): if ( @@ -925,14 +1045,22 @@ def _cluster_duplicates( continue group = [track] used.add(track.path) - for other in bucket_tracks[idx + 1 :]: + candidate_paths: set[str] = set() + for key in track_keys.get(track.path, []): + candidate_paths.update(coarse_index.get(key, set())) + if not candidate_paths: + candidate_paths.update(t.path for t in bucket_tracks[idx + 1 :]) + for other_path in sorted(candidate_paths, key=lambda p: path_order.get(p, len(bucket_tracks))): if cancel_event.is_set() or comparisons >= max_comparisons: review_flags.append("Comparison budget reached; grouping may be incomplete.") break if timeout_sec and (_now() - start_time) > timeout_sec: review_flags.append("Consolidation planning timed out while grouping.") break + if other_path in used or path_order.get(other_path, 0) <= idx: + continue comparisons += 1 + other = path_to_track[other_path] dist = fingerprint_distance(track.fingerprint, other.fingerprint) if dist <= threshold: group.append(other) diff --git a/tests/test_duplicate_consolidation_hardening.py b/tests/test_duplicate_consolidation_hardening.py index bfa4438..f6b6323 100644 --- a/tests/test_duplicate_consolidation_hardening.py +++ b/tests/test_duplicate_consolidation_hardening.py @@ -187,3 +187,65 @@ def test_truncation_sets_review_required_count(tmp_path): assert plan.review_flags assert plan.requires_review assert plan.review_required_count >= 1 + + +def test_coarse_fingerprint_gate_allows_cross_codec_matches(tmp_path): + flac = tmp_path / "Song.flac" + m4a = tmp_path / "Song.m4a" + flac.write_text("a") + m4a.write_text("b") + tracks = [ + { + "path": str(flac), + "fingerprint": " ".join(str(v) for v in range(40)), + "ext": ".flac", + "tags": {"artist": "Artist", "title": "Song"}, + }, + { + "path": str(m4a), + "fingerprint": " ".join([str(v) for v in range(39)] + ["99"]), + "ext": ".m4a", + "tags": {"artist": "Artist", "title": "Song"}, + }, + ] + + plan = build_consolidation_plan(tracks, distance_threshold=0.2) + assert len(plan.groups) == 1 + group_paths = {plan.groups[0].winner_path, *plan.groups[0].losers} + assert group_paths == {str(flac), str(m4a)} + + +def test_metadata_gate_uses_normalized_titles(tmp_path): + base_a = tmp_path / "Track (Remastered 2014).flac" + base_b = tmp_path / "Track.m4a" + other = tmp_path / "Different.mp3" + for p in (base_a, base_b, other): + p.write_text("x") + fp_a = " ".join(["10"] * 20 + ["12"] * 10) + fp_b = " ".join(["10"] * 20 + ["12"] * 9 + ["11"]) + fp_other = " ".join(["10"] * 30) + tracks = [ + { + "path": str(base_a), + "fingerprint": fp_a, + "ext": ".flac", + "tags": {"artist": "Artist", "title": "Track (Remastered 2014)"}, + }, + { + "path": str(base_b), + "fingerprint": fp_b, + "ext": ".m4a", + "tags": {"artist": "Artist", "title": "Track"}, + }, + { + "path": str(other), + "fingerprint": fp_other, + "ext": ".mp3", + "tags": {"artist": "Artist", "title": "Different Song"}, + }, + ] + + plan = build_consolidation_plan(tracks, distance_threshold=0.2) + assert len(plan.groups) == 1 + group_paths = {plan.groups[0].winner_path, *plan.groups[0].losers} + assert group_paths == {str(base_a), str(base_b)} From 897812b716b6039eb3b65a89c52258ef150f933b Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 22 Dec 2025 19:08:54 -0500 Subject: [PATCH 157/606] Add near-duplicate fingerprint grouping --- duplicate_consolidation.py | 47 +++++++++++++++---- tests/test_duplicate_consolidation_guards.py | 1 + .../test_duplicate_consolidation_hardening.py | 5 +- 3 files changed, 43 insertions(+), 10 deletions(-) diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index af0cb66..92d3129 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -43,7 +43,8 @@ from near_duplicate_detector import fingerprint_distance, _parse_fp as _parse_fingerprint LOSSLESS_EXTS = {".flac", ".wav", ".alac", ".ape", ".aiff", ".aif"} -DEFAULT_DISTANCE_THRESHOLD = 0.0 +EXACT_DUPLICATE_THRESHOLD = 0.02 +NEAR_DUPLICATE_THRESHOLD = 0.10 DEFAULT_MAX_CANDIDATES = 5000 DEFAULT_MAX_COMPARISONS = 50_000 DEFAULT_TIMEOUT_SEC = 15.0 @@ -609,6 +610,7 @@ class GroupPlan: track_quality: Dict[str, Dict[str, object]] group_confidence: str artwork_evidence: List[str] + fingerprint_distances: Dict[str, Dict[str, float]] = field(default_factory=dict) library_state: Dict[str, Dict[str, object]] = field(default_factory=dict) def to_dict(self) -> Dict[str, object]: @@ -638,6 +640,7 @@ def to_dict(self) -> Dict[str, object]: "track_quality": {k: dict(v) for k, v in self.track_quality.items()}, "group_confidence": self.group_confidence, "artwork_evidence": list(self.artwork_evidence), + "fingerprint_distances": {k: dict(v) for k, v in self.fingerprint_distances.items()}, "library_state": {k: dict(v) for k, v in self.library_state.items()}, } @@ -997,7 +1000,8 @@ def _select_overall_artwork_candidate(candidates: Sequence[DuplicateTrack]) -> t def _cluster_duplicates( tracks: Sequence[DuplicateTrack], *, - threshold: float, + exact_duplicate_threshold: float, + near_duplicate_threshold: float, cancel_event: threading.Event, max_comparisons: int, timeout_sec: float, @@ -1005,6 +1009,7 @@ def _cluster_duplicates( start_time: float, review_flags: List[str], ) -> List[List[DuplicateTrack]]: + near_duplicate_threshold = max(near_duplicate_threshold, exact_duplicate_threshold) buckets = _build_metadata_buckets(tracks) bucket_items = sorted(buckets.items(), key=lambda x: x[0]) clusters: List[List[DuplicateTrack]] = [] @@ -1062,7 +1067,17 @@ def _cluster_duplicates( comparisons += 1 other = path_to_track[other_path] dist = fingerprint_distance(track.fingerprint, other.fingerprint) - if dist <= threshold: + if dist > near_duplicate_threshold: + continue + compatible = True + for member in group: + if member.path == other.path: + continue + cmp_dist = fingerprint_distance(member.fingerprint, other.fingerprint) + if cmp_dist > near_duplicate_threshold: + compatible = False + break + if compatible: group.append(other) used.add(other.path) if len(group) > 1: @@ -1077,7 +1092,8 @@ def _cluster_duplicates( def build_consolidation_plan( tracks: Iterable[Mapping[str, object]], *, - distance_threshold: float = DEFAULT_DISTANCE_THRESHOLD, + exact_duplicate_threshold: float = EXACT_DUPLICATE_THRESHOLD, + near_duplicate_threshold: float = NEAR_DUPLICATE_THRESHOLD, max_candidates: int = DEFAULT_MAX_CANDIDATES, max_comparisons: int = DEFAULT_MAX_COMPARISONS, timeout_sec: float = DEFAULT_TIMEOUT_SEC, @@ -1103,7 +1119,8 @@ def build_consolidation_plan( clusters = _cluster_duplicates( normalized, - threshold=distance_threshold, + exact_duplicate_threshold=exact_duplicate_threshold, + near_duplicate_threshold=near_duplicate_threshold, cancel_event=cancel_event, max_comparisons=max_comparisons, timeout_sec=timeout_sec, @@ -1143,10 +1160,14 @@ def build_consolidation_plan( winner_quality = _quality_rationale(winner, runner_up, winner_context) distance_samples: List[float] = [] + pair_distances: Dict[str, Dict[str, float]] = {t.path: {} for t in cluster} missing_fingerprints = any(not t.fingerprint for t in cluster) for idx, t in enumerate(cluster): for other in cluster[idx + 1 :]: - distance_samples.append(fingerprint_distance(t.fingerprint, other.fingerprint)) + dist = fingerprint_distance(t.fingerprint, other.fingerprint) + distance_samples.append(dist) + pair_distances[t.path][other.path] = dist + pair_distances[other.path][t.path] = dist max_distance = max(distance_samples) if distance_samples else (1.0 if missing_fingerprints else 0.0) track_quality: Dict[str, Dict[str, object]] = { @@ -1261,9 +1282,18 @@ def build_consolidation_plan( if missing_fingerprints: group_confidence = "Low (missing fingerprints in cluster)" group_review.append("One or more tracks missing fingerprints; requires review.") - if max_distance > 0: + near_distances = [d for d in distance_samples if d > exact_duplicate_threshold] + if near_distances: + group_confidence = f"Medium (near-duplicate fingerprint distance up to {max_distance:.3f})" + group_review.append( + f"Audio match requires review (max distance {max_distance:.3f}; near-duplicate threshold {near_duplicate_threshold:.3f})." + ) + if max_distance > near_duplicate_threshold: group_confidence = f"Low (max fingerprint distance {max_distance:.3f})" - group_review.append(f"Audio match requires review (max distance {max_distance:.3f}).") + if not missing_fingerprints: + group_review.append( + f"Fingerprint distances exceed near-duplicate threshold ({near_duplicate_threshold:.3f}); grouping may be unsafe." + ) dispositions = {loser: "quarantine" for loser in losers} playlist_map = {loser: winner.path for loser in losers} @@ -1324,6 +1354,7 @@ def build_consolidation_plan( track_quality=track_quality, group_confidence=group_confidence, artwork_evidence=artwork_evidence, + fingerprint_distances=pair_distances, library_state=track_states, ) ) diff --git a/tests/test_duplicate_consolidation_guards.py b/tests/test_duplicate_consolidation_guards.py index d2b375b..3fada0d 100644 --- a/tests/test_duplicate_consolidation_guards.py +++ b/tests/test_duplicate_consolidation_guards.py @@ -47,6 +47,7 @@ def _make_group(tmp_path, *, review_flags=None, disposition="quarantine"): track_quality={str(winner): {}, str(loser): {}}, group_confidence="High", artwork_evidence=[], + fingerprint_distances={}, library_state=snapshot, ) diff --git a/tests/test_duplicate_consolidation_hardening.py b/tests/test_duplicate_consolidation_hardening.py index f6b6323..98416b8 100644 --- a/tests/test_duplicate_consolidation_hardening.py +++ b/tests/test_duplicate_consolidation_hardening.py @@ -128,6 +128,7 @@ def test_playlist_rewrite_validation_dry_run(tmp_path): track_quality={str(winner): {}, str(loser): {}}, group_confidence="High", artwork_evidence=[], + fingerprint_distances={}, library_state=snapshot, ) ] @@ -209,7 +210,7 @@ def test_coarse_fingerprint_gate_allows_cross_codec_matches(tmp_path): }, ] - plan = build_consolidation_plan(tracks, distance_threshold=0.2) + plan = build_consolidation_plan(tracks, near_duplicate_threshold=0.2) assert len(plan.groups) == 1 group_paths = {plan.groups[0].winner_path, *plan.groups[0].losers} assert group_paths == {str(flac), str(m4a)} @@ -245,7 +246,7 @@ def test_metadata_gate_uses_normalized_titles(tmp_path): }, ] - plan = build_consolidation_plan(tracks, distance_threshold=0.2) + plan = build_consolidation_plan(tracks, near_duplicate_threshold=0.2) assert len(plan.groups) == 1 group_paths = {plan.groups[0].winner_path, *plan.groups[0].losers} assert group_paths == {str(base_a), str(base_b)} From 3152a29e97b5cc098b09fe28b923b9d63e519376 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Thu, 25 Dec 2025 14:59:37 -0500 Subject: [PATCH 158/606] Gate near-duplicate handling behind review --- .../indexer_engine/music_indexer_api.py | 10 + .../indexer_engine/near_duplicate_detector.py | 174 +++++++++++++++--- music_indexer_api.py | 10 + near_duplicate_detector.py | 174 +++++++++++++++--- tests/test_near_duplicate_review.py | 27 +++ 5 files changed, 353 insertions(+), 42 deletions(-) create mode 100644 tests/test_near_duplicate_review.py diff --git a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py index bb64d5e..2426ec5 100644 --- a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py +++ b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py @@ -459,6 +459,16 @@ def _compute(path: str) -> tuple[int | None, str | None]: coord, max_workers, ) + if near_dupes.review_required: + log_callback( + f" ! Near-duplicate candidates require review ({len(near_dupes.review_groups)} group(s)); " + "no automatic merges will be applied." + ) + for group in near_dupes.review_groups: + log_callback( + f" - Review {os.path.basename(group.winner)} vs {len(group.losers)} candidates " + f"(max fp distance {group.max_distance:.3f})" + ) for loser, reason in near_dupes.items(): if loser not in to_delete: to_delete[loser] = reason diff --git a/library_sync_indexer_engine/indexer_engine/near_duplicate_detector.py b/library_sync_indexer_engine/indexer_engine/near_duplicate_detector.py index 214b3e2..22d4d4e 100644 --- a/library_sync_indexer_engine/indexer_engine/near_duplicate_detector.py +++ b/library_sync_indexer_engine/indexer_engine/near_duplicate_detector.py @@ -3,17 +3,45 @@ from __future__ import annotations import os +import re +import hashlib +from dataclasses import dataclass from collections import defaultdict -from typing import Dict, Set, List, Iterable, Tuple +from typing import Dict, Set, List, Iterable, Tuple, Mapping from concurrent.futures import ThreadPoolExecutor, as_completed import logging import base64 +from itertools import combinations logger = logging.getLogger(__name__) from music_indexer_api import _keep_score EXCLUSION_KEYWORDS = ['remix', 'remastered', 'edit', 'version'] +COARSE_FP_BAND_SIZE = 8 +COARSE_FP_BANDS = 6 +COARSE_FP_QUANTIZATION = 4 + + +@dataclass +class NearDuplicateGroup: + winner: str + losers: List[str] + max_distance: float + reason: str + + +@dataclass +class NearDuplicateResult: + auto_deletes: Dict[str, str] + review_groups: List[NearDuplicateGroup] + + @property + def review_required(self) -> bool: + return bool(self.review_groups) + + def items(self): + return self.auto_deletes.items() def _parse_fp(fp: str) -> tuple[str, List[int] | bytes] | None: @@ -77,22 +105,101 @@ def _has_exclusion(info: Dict[str, str | None]) -> bool: return any(k in text for k in EXCLUSION_KEYWORDS) +def _normalized(text: str | None) -> str: + if not text: + return "" + return " ".join(re.findall(r"[a-z0-9]+", text.lower())) + + +def _metadata_key(info: Mapping[str, str | None]) -> tuple[str, str, str]: + return ( + _normalized(info.get("title")), + _normalized(info.get("primary") or info.get("artist")), + _normalized(info.get("album")), + ) + + +def _metadata_gate(info_a: Mapping[str, str | None], info_b: Mapping[str, str | None]) -> bool: + title_a, primary_a, album_a = _metadata_key(info_a) + title_b, primary_b, album_b = _metadata_key(info_b) + if not title_a or not title_b or title_a != title_b: + return False + if primary_a and primary_b and primary_a == primary_b: + return True + if album_a and album_b and album_a == album_b: + return True + return False + + +def _coarse_fingerprint_keys(fp: str | None) -> List[str]: + parsed = _parse_fp(fp) if fp else None + values: List[int] = [] + if parsed: + kind, payload = parsed + if kind == "ints": + values = list(payload) + elif kind == "bytes": + values = list(payload) + if not values and fp: + values = [ord(ch) for ch in fp] + if not values: + return [] + quantized = [val // COARSE_FP_QUANTIZATION for val in values] + keys: List[str] = [] + limit = COARSE_FP_BAND_SIZE * COARSE_FP_BANDS + for idx in range(0, min(len(quantized), limit), COARSE_FP_BAND_SIZE): + band = quantized[idx : idx + COARSE_FP_BAND_SIZE] + if not band: + break + digest = hashlib.blake2s(",".join(map(str, band)).encode("utf-8"), digest_size=4).hexdigest() + keys.append(f"{idx // COARSE_FP_BAND_SIZE}:{digest}") + if not keys and fp: + keys.append(f"band0:{hashlib.blake2s(fp.encode('utf-8'), digest_size=4).hexdigest()}") + return keys + + +def _coarse_gate(keys_a: List[str], keys_b: List[str]) -> bool: + if not keys_a or not keys_b: + return False + return bool(set(keys_a) & set(keys_b)) + + +def _cluster_max_distance(cluster: Set[str], distances: Dict[frozenset[str], float]) -> float: + pairs = [ + distances[frozenset({a, b})] + for a, b in combinations(sorted(cluster), 2) + if frozenset({a, b}) in distances + ] + return max(pairs) if pairs else 0.0 + + def _scan_paths( paths: Iterable[str], file_infos: Dict[str, Dict[str, str | None]], threshold: float, log_callback, -) -> Tuple[List[Set[str]], int]: +) -> Tuple[List[Tuple[Set[str], float]], int, int, int]: """Scan a list of paths for near duplicate clusters.""" adj: Dict[str, Set[str]] = defaultdict(set) comparisons = 0 + metadata_blocked = 0 + coarse_blocked = 0 + coarse_keys = {p: _coarse_fingerprint_keys(file_infos[p].get("fp")) for p in paths} + pair_distances: Dict[frozenset[str], float] = {} path_list = list(paths) for i, p in enumerate(path_list): for q in path_list[i + 1 :]: + if not _metadata_gate(file_infos[p], file_infos[q]): + metadata_blocked += 1 + continue + if not _coarse_gate(coarse_keys.get(p, []), coarse_keys.get(q, [])): + coarse_blocked += 1 + continue dist = fingerprint_distance(file_infos[p]["fp"], file_infos[q]["fp"]) if dist <= threshold: adj[p].add(q) adj[q].add(p) + pair_distances[frozenset({p, q})] = dist log_callback( f" → near-dup {os.path.basename(p)} vs {os.path.basename(q)} dist={dist:.3f}" ) @@ -114,8 +221,9 @@ def _scan_paths( comp.add(nb) stack.append(nb) if len(comp) > 1: - clusters.append(comp) - return clusters, comparisons + max_distance = _cluster_max_distance(comp, pair_distances) + clusters.append((comp, max_distance)) + return clusters, comparisons, metadata_blocked, coarse_blocked @@ -126,10 +234,10 @@ def _scan_album( file_infos: Dict[str, Dict[str, str | None]], threshold: float, log_callback, -) -> tuple[str, List[Set[str]], int]: +) -> tuple[str, List[Tuple[Set[str], float]], int, int, int]: """Worker helper: scan a single album for near duplicates.""" - clusters, comparisons = _scan_paths(paths, file_infos, threshold, log_callback) - return album, clusters, comparisons + clusters, comparisons, meta_blocked, coarse_blocked = _scan_paths(paths, file_infos, threshold, log_callback) + return album, clusters, comparisons, meta_blocked, coarse_blocked def find_near_duplicates( @@ -140,8 +248,8 @@ def find_near_duplicates( enable_cross_album: bool = False, coord=None, max_workers: int | None = None, -) -> Dict[str, str]: - """Return mapping of files to delete as near duplicates.""" +) -> NearDuplicateResult: + """Return reviewable near-duplicate groups (auto-merging is gated by review).""" if log_callback is None: def log_callback(msg: str) -> None: pass @@ -156,7 +264,9 @@ def log_callback(msg: str) -> None: album_items = sorted(by_album.items(), key=lambda x: x[0]) total_albums = len(album_items) total_comparisons = 0 - clusters: List[Set[str]] = [] + metadata_blocked = 0 + coarse_blocked = 0 + clusters: List[Tuple[Set[str], float]] = [] html_lines = ["

    Phase B – Album Near-Duplicates

    ", "
    "]
     
         with ThreadPoolExecutor(max_workers=max_workers) as ex:
    @@ -173,11 +283,13 @@ def log_callback(msg: str) -> None:
             for fut in as_completed(futures):
                 album = futures[fut]
                 try:
    -                _alb, c, comps = fut.result()
    +                _alb, c, comps, meta_block, coarse_block = fut.result()
                     clusters.extend(c)
                     total_comparisons += comps
    +                metadata_blocked += meta_block
    +                coarse_blocked += coarse_block
                     if coord is not None and c:
    -                    coord.add_near_dupe_clusters([list(s) for s in c])
    +                    coord.add_near_dupe_clusters([list(s[0]) for s in c])
                 except Exception as e:
                     logger.error(f"Phase B album '{album}' failed: {e}")
                     html_lines.append(f"Error scanning album '{album}'")
    @@ -187,7 +299,7 @@ def log_callback(msg: str) -> None:
         if coord is not None:
             coord.set_html_section("B", html_str)
     
    -    cross_clusters: List[Set[str]] = []
    +    cross_clusters: List[Tuple[Set[str], float]] = []
         if enable_cross_album:
             cross_lines = ["

    Phase C – Cross-Album Near-Duplicates

    ", "
    "]
             by_song: Dict[Tuple[str, str], List[str]] = defaultdict(list)
    @@ -214,11 +326,13 @@ def log_callback(msg: str) -> None:
                 for fut in as_completed(futures):
                     _key = futures[fut]
                     try:
    -                    c, comps = fut.result()
    +                    c, comps, meta_block, coarse_block = fut.result()
                         cross_clusters.extend(c)
                         if coord is not None and c:
    -                        coord.add_near_dupe_clusters([list(s) for s in c])
    +                        coord.add_near_dupe_clusters([list(s[0]) for s in c])
                         total_comparisons += comps
    +                    metadata_blocked += meta_block
    +                    coarse_blocked += coarse_block
                     except Exception as e:
                         logger.error(f"Phase C group '{_key}' failed: {e}")
                         cross_lines.append(f"Error scanning group '{_key}'")
    @@ -233,10 +347,16 @@ def log_callback(msg: str) -> None:
                 f"Phase B summary: {total_albums} albums scanned, {total_comparisons} comparisons, {len(clusters)} clusters"
             )
     
    +    log_callback(
    +        "   • Near-duplicate gating summary: "
    +        f"{metadata_blocked} metadata comparisons blocked, {coarse_blocked} coarse fingerprint comparisons blocked "
    +        "(duration/tempo normalization deferred)."
    +    )
    +
         all_clusters = clusters + cross_clusters
     
    -    to_delete: Dict[str, str] = {}
    -    for cluster in all_clusters:
    +    result = NearDuplicateResult(auto_deletes={}, review_groups=[])
    +    for cluster, max_distance in all_clusters:
             by_alb: Dict[str, List[str]] = defaultdict(list)
             for path in cluster:
                 album = file_infos[path].get("album") or ""
    @@ -250,7 +370,19 @@ def log_callback(msg: str) -> None:
                     reverse=True,
                 )
                 keep = scored[0]
    -            for loser in scored[1:]:
    -                if loser not in to_delete:
    -                    to_delete[loser] = f"Near-duplicate of {os.path.basename(keep)}"
    -    return to_delete
    +            losers = scored[1:]
    +            if not losers:
    +                continue
    +            reason = (
    +                f"Near-duplicate of {os.path.basename(keep)} "
    +                f"(max fingerprint distance {max_distance:.3f}); review required"
    +            )
    +            result.review_groups.append(
    +                NearDuplicateGroup(
    +                    winner=keep,
    +                    losers=losers,
    +                    max_distance=max_distance,
    +                    reason=reason,
    +                )
    +            )
    +    return result
    diff --git a/music_indexer_api.py b/music_indexer_api.py
    index bb64d5e..2426ec5 100644
    --- a/music_indexer_api.py
    +++ b/music_indexer_api.py
    @@ -459,6 +459,16 @@ def _compute(path: str) -> tuple[int | None, str | None]:
             coord,
             max_workers,
         )
    +    if near_dupes.review_required:
    +        log_callback(
    +            f"   ! Near-duplicate candidates require review ({len(near_dupes.review_groups)} group(s)); "
    +            "no automatic merges will be applied."
    +        )
    +        for group in near_dupes.review_groups:
    +            log_callback(
    +                f"     - Review {os.path.basename(group.winner)} vs {len(group.losers)} candidates "
    +                f"(max fp distance {group.max_distance:.3f})"
    +            )
         for loser, reason in near_dupes.items():
             if loser not in to_delete:
                 to_delete[loser] = reason
    diff --git a/near_duplicate_detector.py b/near_duplicate_detector.py
    index 214b3e2..22d4d4e 100644
    --- a/near_duplicate_detector.py
    +++ b/near_duplicate_detector.py
    @@ -3,17 +3,45 @@
     from __future__ import annotations
     
     import os
    +import re
    +import hashlib
    +from dataclasses import dataclass
     from collections import defaultdict
    -from typing import Dict, Set, List, Iterable, Tuple
    +from typing import Dict, Set, List, Iterable, Tuple, Mapping
     from concurrent.futures import ThreadPoolExecutor, as_completed
     import logging
     import base64
    +from itertools import combinations
     
     logger = logging.getLogger(__name__)
     
     from music_indexer_api import _keep_score
     
     EXCLUSION_KEYWORDS = ['remix', 'remastered', 'edit', 'version']
    +COARSE_FP_BAND_SIZE = 8
    +COARSE_FP_BANDS = 6
    +COARSE_FP_QUANTIZATION = 4
    +
    +
    +@dataclass
    +class NearDuplicateGroup:
    +    winner: str
    +    losers: List[str]
    +    max_distance: float
    +    reason: str
    +
    +
    +@dataclass
    +class NearDuplicateResult:
    +    auto_deletes: Dict[str, str]
    +    review_groups: List[NearDuplicateGroup]
    +
    +    @property
    +    def review_required(self) -> bool:
    +        return bool(self.review_groups)
    +
    +    def items(self):
    +        return self.auto_deletes.items()
     
     
     def _parse_fp(fp: str) -> tuple[str, List[int] | bytes] | None:
    @@ -77,22 +105,101 @@ def _has_exclusion(info: Dict[str, str | None]) -> bool:
         return any(k in text for k in EXCLUSION_KEYWORDS)
     
     
    +def _normalized(text: str | None) -> str:
    +    if not text:
    +        return ""
    +    return " ".join(re.findall(r"[a-z0-9]+", text.lower()))
    +
    +
    +def _metadata_key(info: Mapping[str, str | None]) -> tuple[str, str, str]:
    +    return (
    +        _normalized(info.get("title")),
    +        _normalized(info.get("primary") or info.get("artist")),
    +        _normalized(info.get("album")),
    +    )
    +
    +
    +def _metadata_gate(info_a: Mapping[str, str | None], info_b: Mapping[str, str | None]) -> bool:
    +    title_a, primary_a, album_a = _metadata_key(info_a)
    +    title_b, primary_b, album_b = _metadata_key(info_b)
    +    if not title_a or not title_b or title_a != title_b:
    +        return False
    +    if primary_a and primary_b and primary_a == primary_b:
    +        return True
    +    if album_a and album_b and album_a == album_b:
    +        return True
    +    return False
    +
    +
    +def _coarse_fingerprint_keys(fp: str | None) -> List[str]:
    +    parsed = _parse_fp(fp) if fp else None
    +    values: List[int] = []
    +    if parsed:
    +        kind, payload = parsed
    +        if kind == "ints":
    +            values = list(payload)
    +        elif kind == "bytes":
    +            values = list(payload)
    +    if not values and fp:
    +        values = [ord(ch) for ch in fp]
    +    if not values:
    +        return []
    +    quantized = [val // COARSE_FP_QUANTIZATION for val in values]
    +    keys: List[str] = []
    +    limit = COARSE_FP_BAND_SIZE * COARSE_FP_BANDS
    +    for idx in range(0, min(len(quantized), limit), COARSE_FP_BAND_SIZE):
    +        band = quantized[idx : idx + COARSE_FP_BAND_SIZE]
    +        if not band:
    +            break
    +        digest = hashlib.blake2s(",".join(map(str, band)).encode("utf-8"), digest_size=4).hexdigest()
    +        keys.append(f"{idx // COARSE_FP_BAND_SIZE}:{digest}")
    +    if not keys and fp:
    +        keys.append(f"band0:{hashlib.blake2s(fp.encode('utf-8'), digest_size=4).hexdigest()}")
    +    return keys
    +
    +
    +def _coarse_gate(keys_a: List[str], keys_b: List[str]) -> bool:
    +    if not keys_a or not keys_b:
    +        return False
    +    return bool(set(keys_a) & set(keys_b))
    +
    +
    +def _cluster_max_distance(cluster: Set[str], distances: Dict[frozenset[str], float]) -> float:
    +    pairs = [
    +        distances[frozenset({a, b})]
    +        for a, b in combinations(sorted(cluster), 2)
    +        if frozenset({a, b}) in distances
    +    ]
    +    return max(pairs) if pairs else 0.0
    +
    +
     def _scan_paths(
         paths: Iterable[str],
         file_infos: Dict[str, Dict[str, str | None]],
         threshold: float,
         log_callback,
    -) -> Tuple[List[Set[str]], int]:
    +) -> Tuple[List[Tuple[Set[str], float]], int, int, int]:
         """Scan a list of paths for near duplicate clusters."""
         adj: Dict[str, Set[str]] = defaultdict(set)
         comparisons = 0
    +    metadata_blocked = 0
    +    coarse_blocked = 0
    +    coarse_keys = {p: _coarse_fingerprint_keys(file_infos[p].get("fp")) for p in paths}
    +    pair_distances: Dict[frozenset[str], float] = {}
         path_list = list(paths)
         for i, p in enumerate(path_list):
             for q in path_list[i + 1 :]:
    +            if not _metadata_gate(file_infos[p], file_infos[q]):
    +                metadata_blocked += 1
    +                continue
    +            if not _coarse_gate(coarse_keys.get(p, []), coarse_keys.get(q, [])):
    +                coarse_blocked += 1
    +                continue
                 dist = fingerprint_distance(file_infos[p]["fp"], file_infos[q]["fp"])
                 if dist <= threshold:
                     adj[p].add(q)
                     adj[q].add(p)
    +                pair_distances[frozenset({p, q})] = dist
                     log_callback(
                         f"   → near-dup {os.path.basename(p)} vs {os.path.basename(q)} dist={dist:.3f}"
                     )
    @@ -114,8 +221,9 @@ def _scan_paths(
                         comp.add(nb)
                         stack.append(nb)
             if len(comp) > 1:
    -            clusters.append(comp)
    -    return clusters, comparisons
    +            max_distance = _cluster_max_distance(comp, pair_distances)
    +            clusters.append((comp, max_distance))
    +    return clusters, comparisons, metadata_blocked, coarse_blocked
     
     
     
    @@ -126,10 +234,10 @@ def _scan_album(
         file_infos: Dict[str, Dict[str, str | None]],
         threshold: float,
         log_callback,
    -) -> tuple[str, List[Set[str]], int]:
    +) -> tuple[str, List[Tuple[Set[str], float]], int, int, int]:
         """Worker helper: scan a single album for near duplicates."""
    -    clusters, comparisons = _scan_paths(paths, file_infos, threshold, log_callback)
    -    return album, clusters, comparisons
    +    clusters, comparisons, meta_blocked, coarse_blocked = _scan_paths(paths, file_infos, threshold, log_callback)
    +    return album, clusters, comparisons, meta_blocked, coarse_blocked
     
     
     def find_near_duplicates(
    @@ -140,8 +248,8 @@ def find_near_duplicates(
         enable_cross_album: bool = False,
         coord=None,
         max_workers: int | None = None,
    -) -> Dict[str, str]:
    -    """Return mapping of files to delete as near duplicates."""
    +) -> NearDuplicateResult:
    +    """Return reviewable near-duplicate groups (auto-merging is gated by review)."""
         if log_callback is None:
             def log_callback(msg: str) -> None:
                 pass
    @@ -156,7 +264,9 @@ def log_callback(msg: str) -> None:
         album_items = sorted(by_album.items(), key=lambda x: x[0])
         total_albums = len(album_items)
         total_comparisons = 0
    -    clusters: List[Set[str]] = []
    +    metadata_blocked = 0
    +    coarse_blocked = 0
    +    clusters: List[Tuple[Set[str], float]] = []
         html_lines = ["

    Phase B – Album Near-Duplicates

    ", "
    "]
     
         with ThreadPoolExecutor(max_workers=max_workers) as ex:
    @@ -173,11 +283,13 @@ def log_callback(msg: str) -> None:
             for fut in as_completed(futures):
                 album = futures[fut]
                 try:
    -                _alb, c, comps = fut.result()
    +                _alb, c, comps, meta_block, coarse_block = fut.result()
                     clusters.extend(c)
                     total_comparisons += comps
    +                metadata_blocked += meta_block
    +                coarse_blocked += coarse_block
                     if coord is not None and c:
    -                    coord.add_near_dupe_clusters([list(s) for s in c])
    +                    coord.add_near_dupe_clusters([list(s[0]) for s in c])
                 except Exception as e:
                     logger.error(f"Phase B album '{album}' failed: {e}")
                     html_lines.append(f"Error scanning album '{album}'")
    @@ -187,7 +299,7 @@ def log_callback(msg: str) -> None:
         if coord is not None:
             coord.set_html_section("B", html_str)
     
    -    cross_clusters: List[Set[str]] = []
    +    cross_clusters: List[Tuple[Set[str], float]] = []
         if enable_cross_album:
             cross_lines = ["

    Phase C – Cross-Album Near-Duplicates

    ", "
    "]
             by_song: Dict[Tuple[str, str], List[str]] = defaultdict(list)
    @@ -214,11 +326,13 @@ def log_callback(msg: str) -> None:
                 for fut in as_completed(futures):
                     _key = futures[fut]
                     try:
    -                    c, comps = fut.result()
    +                    c, comps, meta_block, coarse_block = fut.result()
                         cross_clusters.extend(c)
                         if coord is not None and c:
    -                        coord.add_near_dupe_clusters([list(s) for s in c])
    +                        coord.add_near_dupe_clusters([list(s[0]) for s in c])
                         total_comparisons += comps
    +                    metadata_blocked += meta_block
    +                    coarse_blocked += coarse_block
                     except Exception as e:
                         logger.error(f"Phase C group '{_key}' failed: {e}")
                         cross_lines.append(f"Error scanning group '{_key}'")
    @@ -233,10 +347,16 @@ def log_callback(msg: str) -> None:
                 f"Phase B summary: {total_albums} albums scanned, {total_comparisons} comparisons, {len(clusters)} clusters"
             )
     
    +    log_callback(
    +        "   • Near-duplicate gating summary: "
    +        f"{metadata_blocked} metadata comparisons blocked, {coarse_blocked} coarse fingerprint comparisons blocked "
    +        "(duration/tempo normalization deferred)."
    +    )
    +
         all_clusters = clusters + cross_clusters
     
    -    to_delete: Dict[str, str] = {}
    -    for cluster in all_clusters:
    +    result = NearDuplicateResult(auto_deletes={}, review_groups=[])
    +    for cluster, max_distance in all_clusters:
             by_alb: Dict[str, List[str]] = defaultdict(list)
             for path in cluster:
                 album = file_infos[path].get("album") or ""
    @@ -250,7 +370,19 @@ def log_callback(msg: str) -> None:
                     reverse=True,
                 )
                 keep = scored[0]
    -            for loser in scored[1:]:
    -                if loser not in to_delete:
    -                    to_delete[loser] = f"Near-duplicate of {os.path.basename(keep)}"
    -    return to_delete
    +            losers = scored[1:]
    +            if not losers:
    +                continue
    +            reason = (
    +                f"Near-duplicate of {os.path.basename(keep)} "
    +                f"(max fingerprint distance {max_distance:.3f}); review required"
    +            )
    +            result.review_groups.append(
    +                NearDuplicateGroup(
    +                    winner=keep,
    +                    losers=losers,
    +                    max_distance=max_distance,
    +                    reason=reason,
    +                )
    +            )
    +    return result
    diff --git a/tests/test_near_duplicate_review.py b/tests/test_near_duplicate_review.py
    new file mode 100644
    index 0000000..0cb9f98
    --- /dev/null
    +++ b/tests/test_near_duplicate_review.py
    @@ -0,0 +1,27 @@
    +from dry_run_coordinator import DryRunCoordinator
    +from near_duplicate_detector import find_near_duplicates
    +
    +
    +def test_near_duplicates_marked_for_review():
    +    infos = {
    +        "a1": {"fp": "1 2 3", "album": "A", "title": "Song", "primary": "Artist", "meta_count": 2},
    +        "a2": {"fp": "1 2 3", "album": "A", "title": "Song", "primary": "Artist", "meta_count": 1},
    +    }
    +    result = find_near_duplicates(infos, {".mp3": 0}, 0.5)
    +    assert result.review_required
    +    assert dict(result.items()) == {}
    +    assert result.review_groups
    +    group = result.review_groups[0]
    +    assert set(group.losers) == {"a2"} or set(group.losers) == {"a1"}
    +    assert "review required" in group.reason.lower()
    +
    +
    +def test_metadata_gate_blocks_conflicting_titles():
    +    infos = {
    +        "x1": {"fp": "1 2 3", "album": "Mix", "title": "Song Alpha", "primary": "Artist", "meta_count": 1},
    +        "x2": {"fp": "1 2 3", "album": "Mix", "title": "Song Beta", "primary": "Artist", "meta_count": 1},
    +    }
    +    coord = DryRunCoordinator()
    +    result = find_near_duplicates(infos, {".mp3": 0}, 0.5, coord=coord)
    +    assert not result.review_required
    +    assert not coord.near_dupe_clusters
    
    From 250c3846542cfedb2f0e20c2e95654630c31aca2 Mon Sep 17 00:00:00 2001
    From: Greensand321 
    Date: Thu, 25 Dec 2025 15:07:51 -0500
    Subject: [PATCH 159/606] Clarify duplicate grouping rationale in preview
    
    ---
     duplicate_consolidation.py                    | 222 ++++++++++++++++--
     tests/test_duplicate_consolidation_guards.py  |   4 +
     .../test_duplicate_consolidation_hardening.py |   4 +
     3 files changed, 206 insertions(+), 24 deletions(-)
    
    diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py
    index 92d3129..c60a410 100644
    --- a/duplicate_consolidation.py
    +++ b/duplicate_consolidation.py
    @@ -581,6 +581,48 @@ def to_dict(self) -> Dict[str, int]:
             return {"playlists": self.playlists, "entries": self.entries}
     
     
    +@dataclass
    +class GroupingDecision:
    +    """Evidence for why a candidate joined a duplicate group."""
    +
    +    anchor_path: str
    +    candidate_path: str
    +    metadata_key: tuple[str, str]
    +    coarse_keys_anchor: List[str]
    +    coarse_keys_candidate: List[str]
    +    shared_coarse_keys: List[str]
    +    distance_to_anchor: float
    +    max_group_distance: float
    +    threshold: float
    +    match_type: str
    +    coarse_gate: str
    +
    +    def to_dict(self) -> Dict[str, object]:
    +        return {
    +            "anchor_path": self.anchor_path,
    +            "candidate_path": self.candidate_path,
    +            "metadata_key": list(self.metadata_key),
    +            "coarse_keys_anchor": list(self.coarse_keys_anchor),
    +            "coarse_keys_candidate": list(self.coarse_keys_candidate),
    +            "shared_coarse_keys": list(self.shared_coarse_keys),
    +            "distance_to_anchor": self.distance_to_anchor,
    +            "max_group_distance": self.max_group_distance,
    +            "threshold": self.threshold,
    +            "match_type": self.match_type,
    +            "coarse_gate": self.coarse_gate,
    +        }
    +
    +
    +@dataclass
    +class ClusterResult:
    +    """Grouping outcome with evidence for why tracks clustered."""
    +
    +    tracks: List[DuplicateTrack]
    +    metadata_key: tuple[str, str]
    +    decisions: List[GroupingDecision]
    +    exact_threshold: float
    +    near_threshold: float
    +
     @dataclass
     class GroupPlan:
         """Planned actions for a duplicate group."""
    @@ -609,6 +651,10 @@ class GroupPlan:
         tag_source_evidence: List[str]
         track_quality: Dict[str, Dict[str, object]]
         group_confidence: str
    +    group_match_type: str
    +    grouping_metadata_key: tuple[str, str]
    +    grouping_thresholds: Dict[str, float]
    +    grouping_decisions: List[GroupingDecision]
         artwork_evidence: List[str]
         fingerprint_distances: Dict[str, Dict[str, float]] = field(default_factory=dict)
         library_state: Dict[str, Dict[str, object]] = field(default_factory=dict)
    @@ -639,6 +685,10 @@ def to_dict(self) -> Dict[str, object]:
                 "tag_source_evidence": list(self.tag_source_evidence),
                 "track_quality": {k: dict(v) for k, v in self.track_quality.items()},
                 "group_confidence": self.group_confidence,
    +            "group_match_type": self.group_match_type,
    +            "grouping_metadata_key": list(self.grouping_metadata_key),
    +            "grouping_thresholds": dict(self.grouping_thresholds),
    +            "grouping_decisions": [d.to_dict() for d in self.grouping_decisions],
                 "artwork_evidence": list(self.artwork_evidence),
                 "fingerprint_distances": {k: dict(v) for k, v in self.fingerprint_distances.items()},
                 "library_state": {k: dict(v) for k, v in self.library_state.items()},
    @@ -1008,14 +1058,14 @@ def _cluster_duplicates(
         progress_callback: Callable[[int, int, str], None],
         start_time: float,
         review_flags: List[str],
    -) -> List[List[DuplicateTrack]]:
    +) -> List[ClusterResult]:
         near_duplicate_threshold = max(near_duplicate_threshold, exact_duplicate_threshold)
         buckets = _build_metadata_buckets(tracks)
         bucket_items = sorted(buckets.items(), key=lambda x: x[0])
    -    clusters: List[List[DuplicateTrack]] = []
    +    clusters: List[ClusterResult] = []
         comparisons = 0
         processed = 0
    -    for _, bucket_tracks in bucket_items:
    +    for bucket_key, bucket_tracks in bucket_items:
             if cancel_event.is_set() or (timeout_sec and (_now() - start_time) > timeout_sec):
                 review_flags.append("Consolidation planning cancelled or timed out during grouping.")
                 break
    @@ -1049,12 +1099,16 @@ def _cluster_duplicates(
                 if track.path in used:
                     continue
                 group = [track]
    +            decisions: List[GroupingDecision] = []
                 used.add(track.path)
                 candidate_paths: set[str] = set()
                 for key in track_keys.get(track.path, []):
                     candidate_paths.update(coarse_index.get(key, set()))
                 if not candidate_paths:
    +                fallback_candidates = True
                     candidate_paths.update(t.path for t in bucket_tracks[idx + 1 :])
    +            else:
    +                fallback_candidates = False
                 for other_path in sorted(candidate_paths, key=lambda p: path_order.get(p, len(bucket_tracks))):
                     if cancel_event.is_set() or comparisons >= max_comparisons:
                         review_flags.append("Comparison budget reached; grouping may be incomplete.")
    @@ -1070,18 +1124,48 @@ def _cluster_duplicates(
                     if dist > near_duplicate_threshold:
                         continue
                     compatible = True
    +                max_candidate_distance = dist
                     for member in group:
                         if member.path == other.path:
                             continue
                         cmp_dist = fingerprint_distance(member.fingerprint, other.fingerprint)
    +                    max_candidate_distance = max(max_candidate_distance, cmp_dist)
                         if cmp_dist > near_duplicate_threshold:
                             compatible = False
                             break
                     if compatible:
    +                    anchor_keys = track_keys.get(track.path, [])
    +                    candidate_keys = track_keys.get(other.path, [])
    +                    shared_keys = sorted(set(anchor_keys) & set(candidate_keys))
    +                    coarse_gate = "match" if shared_keys else "fallback (no shared coarse keys)" if fallback_candidates else "none"
    +                    match_type = "exact" if max_candidate_distance <= exact_duplicate_threshold else "near"
    +                    decisions.append(
    +                        GroupingDecision(
    +                            anchor_path=track.path,
    +                            candidate_path=other.path,
    +                            metadata_key=bucket_key,
    +                            coarse_keys_anchor=anchor_keys,
    +                            coarse_keys_candidate=candidate_keys,
    +                            shared_coarse_keys=shared_keys,
    +                            distance_to_anchor=dist,
    +                            max_group_distance=max_candidate_distance,
    +                            threshold=near_duplicate_threshold,
    +                            match_type=match_type,
    +                            coarse_gate=coarse_gate,
    +                        )
    +                    )
                         group.append(other)
                         used.add(other.path)
                 if len(group) > 1:
    -                clusters.append(group)
    +                clusters.append(
    +                    ClusterResult(
    +                        tracks=group,
    +                        metadata_key=bucket_key,
    +                        decisions=decisions,
    +                        exact_threshold=exact_duplicate_threshold,
    +                        near_threshold=near_duplicate_threshold,
    +                    )
    +                )
                 processed += 1
                 progress_callback(processed, len(bucket_tracks), track.path)
             if cancel_event.is_set() or comparisons >= max_comparisons or (timeout_sec and (_now() - start_time) > timeout_sec):
    @@ -1131,21 +1215,22 @@ def build_consolidation_plan(
     
         plans: List[GroupPlan] = []
         plan_placeholders = False
    -    for cluster in sorted(clusters, key=lambda c: _stable_group_id([t.path for t in c])):
    +    for cluster in sorted(clusters, key=lambda c: _stable_group_id([t.path for t in c.tracks])):
    +        cluster_tracks = cluster.tracks
             contexts: Dict[str, str] = {}
             context_evidence: Dict[str, List[str]] = {}
    -        for t in cluster:
    +        for t in cluster_tracks:
                 ctx, evidence = _classify_context(t)
                 contexts[t.path] = ctx
                 t.context_evidence = evidence
                 context_evidence[t.path] = evidence
     
    -        quality_sorted = sorted(cluster, key=lambda t: _quality_tuple(t, contexts[t.path]), reverse=True)
    +        quality_sorted = sorted(cluster_tracks, key=lambda t: _quality_tuple(t, contexts[t.path]), reverse=True)
             winner = quality_sorted[0]
             losers = [t.path for t in quality_sorted[1:]]
             runner_up = quality_sorted[1] if len(quality_sorted) > 1 else None
     
    -        metadata_source = _select_metadata_source(cluster, contexts)
    +        metadata_source = _select_metadata_source(cluster_tracks, contexts)
             planned_tags, tag_source, tag_source_reason, tag_source_evidence = _build_planned_tags(
                 winner, metadata_source, contexts
             )
    @@ -1160,10 +1245,10 @@ def build_consolidation_plan(
             winner_quality = _quality_rationale(winner, runner_up, winner_context)
     
             distance_samples: List[float] = []
    -        pair_distances: Dict[str, Dict[str, float]] = {t.path: {} for t in cluster}
    -        missing_fingerprints = any(not t.fingerprint for t in cluster)
    -        for idx, t in enumerate(cluster):
    -            for other in cluster[idx + 1 :]:
    +        pair_distances: Dict[str, Dict[str, float]] = {t.path: {} for t in cluster_tracks}
    +        missing_fingerprints = any(not t.fingerprint for t in cluster_tracks)
    +        for idx, t in enumerate(cluster_tracks):
    +            for other in cluster_tracks[idx + 1 :]:
                     dist = fingerprint_distance(t.fingerprint, other.fingerprint)
                     distance_samples.append(dist)
                     pair_distances[t.path][other.path] = dist
    @@ -1183,8 +1268,12 @@ def build_consolidation_plan(
                 for t in quality_sorted
             }
     
    -        single_art_track, single_art_blob, ambiguous_single_art = _select_single_artwork_candidate(cluster, contexts)
    -        overall_art_track, overall_art_blob, ambiguous_overall_art = _select_overall_artwork_candidate(cluster)
    +        single_art_track, single_art_blob, ambiguous_single_art = _select_single_artwork_candidate(
    +            cluster_tracks, contexts
    +        )
    +        overall_art_track, overall_art_blob, ambiguous_overall_art = _select_overall_artwork_candidate(
    +            cluster_tracks
    +        )
             artwork_actions: List[ArtworkDirective] = []
             chosen_artwork_source: Dict[str, object] = {"path": None, "reason": ""}
             artwork_status = "unchanged"
    @@ -1258,7 +1347,7 @@ def build_consolidation_plan(
                 elif not single_art_blob and not overall_art_blob:
                     group_review.append("No artwork available to apply.")
     
    -        cover_hashes = {t.cover_hash for t in cluster if t.cover_hash}
    +        cover_hashes = {t.cover_hash for t in cluster_tracks if t.cover_hash}
             ambiguous_art = ambiguous_single_art or ambiguous_overall_art
             if len(cover_hashes) > 1:
                 ambiguous_art = True
    @@ -1303,25 +1392,27 @@ def build_consolidation_plan(
                 group_review.append("Artwork selection requires review.")
             if not losers:
                 group_review.append("No losers to consolidate.")
    -        if any(_has_review_keyword(t) for t in cluster):
    +        if any(_has_review_keyword(t) for t in cluster_tracks):
                 group_review.append("Contains remix/sped-up variant indicators; review recommended.")
    -        placeholders = _placeholder_present(planned_tags) or any(_placeholder_present(t.current_tags) for t in cluster)
    +        placeholders = _placeholder_present(planned_tags) or any(
    +            _placeholder_present(t.current_tags) for t in cluster_tracks
    +        )
             if placeholders:
                 group_review.append("Placeholder metadata detected; requires review.")
                 plan_placeholders = True
             if winner.metadata_error:
                 group_review.append(f"Metadata read issue for winner: {winner.metadata_error}")
    -        if any(t.artwork_error for t in cluster):
    +        if any(t.artwork_error for t in cluster_tracks):
                 group_review.append("Artwork extraction failed for at least one track.")
     
             playlist_impact = PlaylistImpact(playlists=len(losers), entries=len(losers))
    -        track_states = {t.path: dict(t.library_state) for t in cluster}
    +        track_states = {t.path: dict(t.library_state) for t in cluster_tracks}
     
             artwork_candidates: List[ArtworkCandidate] = []
    -        for t in cluster:
    +        for t in cluster_tracks:
                 artwork_candidates.extend(t.artwork)
     
    -        current_tags = {t.path: _merge_tags(_blank_tags(), t.current_tags) for t in cluster}
    +        current_tags = {t.path: _merge_tags(_blank_tags(), t.current_tags) for t in cluster_tracks}
     
             plans.append(
                 GroupPlan(
    @@ -1342,9 +1433,9 @@ def build_consolidation_plan(
                     playlist_impact=playlist_impact,
                     review_flags=group_review,
                     context_summary={
    -                    "album": sorted([t.path for t in cluster if contexts.get(t.path) == "album"]),
    -                    "single": sorted([t.path for t in cluster if contexts.get(t.path) == "single"]),
    -                    "unknown": sorted([t.path for t in cluster if contexts.get(t.path) == "unknown"]),
    +                    "album": sorted([t.path for t in cluster_tracks if contexts.get(t.path) == "album"]),
    +                    "single": sorted([t.path for t in cluster_tracks if contexts.get(t.path) == "single"]),
    +                    "unknown": sorted([t.path for t in cluster_tracks if contexts.get(t.path) == "unknown"]),
                     },
                     context_evidence=context_evidence,
                     tag_source=tag_source,
    @@ -1353,6 +1444,13 @@ def build_consolidation_plan(
                     tag_source_evidence=tag_source_evidence,
                     track_quality=track_quality,
                     group_confidence=group_confidence,
    +                group_match_type="Exact" if max_distance <= exact_duplicate_threshold else "Near-duplicate",
    +                grouping_metadata_key=cluster.metadata_key,
    +                grouping_thresholds={
    +                    "exact": cluster.exact_threshold,
    +                    "near": cluster.near_threshold,
    +                },
    +                grouping_decisions=cluster.decisions,
                     artwork_evidence=artwork_evidence,
                     fingerprint_distances=pair_distances,
                     library_state=track_states,
    @@ -1418,6 +1516,78 @@ def _render_quality(group: GroupPlan) -> str:
                 ""
             )
     
    +    def _render_codec_mix_warning(group: GroupPlan) -> str:
    +        labels: List[str] = []
    +        for path, quality in group.track_quality.items():
    +            container = str(quality.get("container") or "").strip()
    +            if not container:
    +                ext = os.path.splitext(path)[1].replace(".", "").upper()
    +                container = ext or "UNKNOWN"
    +            labels.append(container.upper())
    +        unique = sorted(set(labels))
    +        if len(unique) <= 1:
    +            return ""
    +        return f"

    Codec mix warning: {' + '.join(unique)} in group

    " + + def _render_grouping_evidence(group: GroupPlan) -> str: + meta_artist, meta_title = group.grouping_metadata_key + lines = [ + "

    Grouping rationale

    ", + f"

    Metadata gate: artist '{_html_escape(meta_artist)}', title '{_html_escape(meta_title)}'

    ", + f"

    Fingerprint thresholds: exact ≤ {_html_escape(group.grouping_thresholds.get('exact'))}, " + f"near ≤ {_html_escape(group.grouping_thresholds.get('near'))}

    ", + f"

    Grouping type: {_html_escape(group.group_match_type)}

    ", + ] + if group.grouping_decisions: + rows = [] + for decision in group.grouping_decisions: + shared = ", ".join(decision.shared_coarse_keys) if decision.shared_coarse_keys else "—" + rows.append( + "" + f"{_html_escape(decision.candidate_path)}" + f"{_html_escape(decision.anchor_path)}" + f"{_html_escape(decision.coarse_gate)}" + f"{_html_escape(shared)}" + f"{decision.distance_to_anchor:.4f}" + f"{decision.max_group_distance:.4f}" + f"{_html_escape(decision.match_type)}" + "" + ) + lines.append( + "" + "" + "" + + "".join(rows) + + "" + ) + else: + lines.append("No grouping decisions recorded.") + if group.fingerprint_distances: + rows = [] + for left, right_map in sorted(group.fingerprint_distances.items()): + for right, dist in sorted(right_map.items()): + if left >= right: + continue + match_type = "exact" if dist <= group.grouping_thresholds.get("exact", 0.0) else "near" + rows.append( + "" + f"{_html_escape(left)}" + f"{_html_escape(right)}" + f"{dist:.4f}" + f"{_html_escape(match_type)}" + "" + ) + if rows: + lines.append( + "

    Fingerprint distances

    " + "" + "" + + "".join(rows) + + "" + ) + lines.append("
    ") + return "".join(lines) + def _render_losers(group: GroupPlan) -> str: if not group.losers: return "No losers to consolidate." @@ -1504,8 +1674,12 @@ def _render_artwork_section(group: GroupPlan) -> str: f"
    Group {group.group_id}" f"
    Confidence: {_html_escape(group.group_confidence)}
    {review_badge}
    " ) + codec_warning = _render_codec_mix_warning(group) + if codec_warning: + lines.append(codec_warning) lines.append(f"

    Winner: {_html_escape(group.winner_path)}

    ") lines.append(_render_quality(group)) + lines.append(_render_grouping_evidence(group)) lines.append("

    Losers

    ") lines.append(_render_losers(group)) diff --git a/tests/test_duplicate_consolidation_guards.py b/tests/test_duplicate_consolidation_guards.py index 3fada0d..4527e2c 100644 --- a/tests/test_duplicate_consolidation_guards.py +++ b/tests/test_duplicate_consolidation_guards.py @@ -46,6 +46,10 @@ def _make_group(tmp_path, *, review_flags=None, disposition="quarantine"): tag_source_evidence=[], track_quality={str(winner): {}, str(loser): {}}, group_confidence="High", + group_match_type="Exact", + grouping_metadata_key=("artist", "title"), + grouping_thresholds={"exact": 0.01, "near": 0.03}, + grouping_decisions=[], artwork_evidence=[], fingerprint_distances={}, library_state=snapshot, diff --git a/tests/test_duplicate_consolidation_hardening.py b/tests/test_duplicate_consolidation_hardening.py index 98416b8..8f1823b 100644 --- a/tests/test_duplicate_consolidation_hardening.py +++ b/tests/test_duplicate_consolidation_hardening.py @@ -127,6 +127,10 @@ def test_playlist_rewrite_validation_dry_run(tmp_path): tag_source_evidence=[], track_quality={str(winner): {}, str(loser): {}}, group_confidence="High", + group_match_type="Exact", + grouping_metadata_key=("artist", "title"), + grouping_thresholds={"exact": 0.01, "near": 0.03}, + grouping_decisions=[], artwork_evidence=[], fingerprint_distances={}, library_state=snapshot, From 9db0fc8afacb84d06bc5d995e92f41fb324cf2c7 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Thu, 25 Dec 2025 15:27:15 -0500 Subject: [PATCH 160/606] Fix consolidation plan group id --- duplicate_consolidation.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index c60a410..c6aaad3 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -1353,12 +1353,14 @@ def build_consolidation_plan( ambiguous_art = True artwork_evidence.append("Conflicting embedded artwork hashes between candidates.") if ambiguous_art: - review_flags.append(f"Artwork selection ambiguous for group {_stable_group_id([t.path for t in cluster])}.") + review_flags.append( + f"Artwork selection ambiguous for group {_stable_group_id([t.path for t in cluster_tracks])}." + ) if not (single_art_blob or overall_art_blob): missing_reason = "No artwork candidates available" - if any(t.artwork_error for t in cluster): + if any(t.artwork_error for t in cluster_tracks): missing_reason = "Artwork present but unreadable" - review_flags.append(f"{missing_reason} for group {_stable_group_id([t.path for t in cluster])}.") + review_flags.append(f"{missing_reason} for group {_stable_group_id([t.path for t in cluster_tracks])}.") chosen_artwork_source["reason"] = missing_reason artwork_status = "none found" group_review.append(missing_reason) @@ -1387,7 +1389,7 @@ def build_consolidation_plan( dispositions = {loser: "quarantine" for loser in losers} playlist_map = {loser: winner.path for loser in losers} - group_id = _stable_group_id([t.path for t in cluster]) + group_id = _stable_group_id([t.path for t in cluster_tracks]) if ambiguous_art: group_review.append("Artwork selection requires review.") if not losers: From 7cc6dc8070af6b6129a0c13dec78fe80c8aa6b83 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Fri, 26 Dec 2025 09:13:56 -0500 Subject: [PATCH 161/606] Make consolidation release-aware --- duplicate_consolidation.py | 550 ++++++++++++++++++++++--------------- 1 file changed, 331 insertions(+), 219 deletions(-) diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index c6aaad3..23b292c 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -165,6 +165,44 @@ def _normalize_title(tags: Mapping[str, object], path: str) -> str: return " ".join(filtered or tokens or ["unknown"]) +def _normalize_album_key(tags: Mapping[str, object], path: str) -> tuple[str, str] | None: + raw_album = str(tags.get("album") or "").strip() + if not raw_album: + return None + album_tokens = _normalized_tokens(raw_album) + if not album_tokens: + return None + album_title = " ".join(album_tokens) + album_artist = _normalize_primary_artist(tags, path) + if album_artist == "unknown": + album_artist = "" + return album_artist, album_title + + +def _album_display_label(tags: Mapping[str, object]) -> str: + album = str(tags.get("album") or "").strip() + artist = str(tags.get("albumartist") or tags.get("artist") or "").strip() + if album and artist: + return f"{artist} — {album}" + return album or artist or "Unknown album" + + +def _build_album_track_counts(tracks: Sequence["DuplicateTrack"]) -> Dict[tuple[str, str], int]: + album_titles: Dict[tuple[str, str], set[str]] = defaultdict(set) + for track in tracks: + context, _ = _classify_context(track) + if context == "unknown": + continue + tags = track.current_tags if isinstance(track.current_tags, Mapping) else track.tags + tags = tags or {} + album_key = _normalize_album_key(tags, track.path) + if not album_key: + continue + title_key = _normalize_title(tags, track.path) + album_titles[album_key].add(title_key) + return {key: len(titles) for key, titles in album_titles.items()} + + def _metadata_bucket_key(track: DuplicateTrack) -> tuple[str, str]: tags = track.current_tags if isinstance(track.current_tags, Mapping) else track.tags tags = tags or {} @@ -1201,6 +1239,8 @@ def build_consolidation_plan( review_flags.append(f"Truncated candidate set to {max_candidates} items to protect runtime.") break + album_track_counts = _build_album_track_counts(normalized) + clusters = _cluster_duplicates( normalized, exact_duplicate_threshold=exact_duplicate_threshold, @@ -1225,239 +1265,311 @@ def build_consolidation_plan( t.context_evidence = evidence context_evidence[t.path] = evidence - quality_sorted = sorted(cluster_tracks, key=lambda t: _quality_tuple(t, contexts[t.path]), reverse=True) - winner = quality_sorted[0] - losers = [t.path for t in quality_sorted[1:]] - runner_up = quality_sorted[1] if len(quality_sorted) > 1 else None + album_key_by_path: Dict[str, tuple[str, str] | None] = {} + release_by_path: Dict[str, str] = {} + album_label_by_key: Dict[tuple[str, str], str] = {} + for t in cluster_tracks: + tags = t.current_tags if isinstance(t.current_tags, Mapping) else t.tags + tags = tags or {} + album_key = _normalize_album_key(tags, t.path) + album_key_by_path[t.path] = album_key + if album_key and album_key not in album_label_by_key: + album_label_by_key[album_key] = _album_display_label(tags) + context = contexts.get(t.path, "unknown") + if context == "unknown" or not album_key: + release_by_path[t.path] = "unknown" + continue + album_count = album_track_counts.get(album_key, 0) + if album_count >= 2: + release_by_path[t.path] = "album" + elif album_count == 1: + release_by_path[t.path] = "single" + else: + release_by_path[t.path] = "unknown" - metadata_source = _select_metadata_source(cluster_tracks, contexts) - planned_tags, tag_source, tag_source_reason, tag_source_evidence = _build_planned_tags( - winner, metadata_source, contexts - ) - if not metadata_source: - review_flags.append(f"Missing metadata source for group containing {winner.path}.") - group_review = ["Metadata source missing; tags may be incomplete."] + album_groups: Dict[tuple[str, str], List[DuplicateTrack]] = defaultdict(list) + single_tracks: List[DuplicateTrack] = [] + unknown_tracks: List[DuplicateTrack] = [] + for t in cluster_tracks: + release_type = release_by_path.get(t.path, "unknown") + if release_type == "album" and album_key_by_path.get(t.path): + album_groups[album_key_by_path[t.path]].append(t) + elif release_type == "single": + single_tracks.append(t) + else: + unknown_tracks.append(t) + + release_groups: List[tuple[List[DuplicateTrack], List[str]]] = [] + if len(album_groups) > 1: + album_labels = [ + album_label_by_key.get(key, "Unknown album") for key in sorted(album_groups.keys()) + ] + album_summary = ", ".join(sorted({label for label in album_labels if label})) + cross_album_note = "Identical recording across albums; preserving tracks for distinct releases." + if album_summary: + cross_album_note = f"{cross_album_note} Albums: {album_summary}." + for album_key, tracks in sorted( + album_groups.items(), key=lambda item: album_label_by_key.get(item[0], "") + ): + if tracks: + release_groups.append((tracks, [cross_album_note])) + if single_tracks: + release_groups.append( + ( + single_tracks, + [ + cross_album_note, + "Singles consolidated separately from multi-album matches.", + ], + ) + ) + if unknown_tracks: + release_groups.append((unknown_tracks, [cross_album_note, "Unknown release context kept separate."])) else: - group_review: List[str] = [] - - meta_changes = _metadata_changes(winner, planned_tags) - winner_context = contexts.get(winner.path, "unknown") - winner_quality = _quality_rationale(winner, runner_up, winner_context) - - distance_samples: List[float] = [] - pair_distances: Dict[str, Dict[str, float]] = {t.path: {} for t in cluster_tracks} - missing_fingerprints = any(not t.fingerprint for t in cluster_tracks) - for idx, t in enumerate(cluster_tracks): - for other in cluster_tracks[idx + 1 :]: - dist = fingerprint_distance(t.fingerprint, other.fingerprint) - distance_samples.append(dist) - pair_distances[t.path][other.path] = dist - pair_distances[other.path][t.path] = dist - max_distance = max(distance_samples) if distance_samples else (1.0 if missing_fingerprints else 0.0) - - track_quality: Dict[str, Dict[str, object]] = { - t.path: { - "container": t.container or t.ext.replace(".", "").upper(), - "codec": t.codec, - "bitrate": t.bitrate, - "sample_rate": t.sample_rate, - "bit_depth": t.bit_depth, - "channels": t.channels, - "is_lossless": t.is_lossless, + release_groups.append((cluster_tracks, [])) + + for group_tracks, release_notes in release_groups: + group_paths = {t.path for t in group_tracks} + group_contexts = {path: contexts[path] for path in group_paths} + group_context_evidence = {path: context_evidence[path] for path in group_paths} + group_decisions = [ + d for d in cluster.decisions if d.anchor_path in group_paths and d.candidate_path in group_paths + ] + + quality_sorted = sorted(group_tracks, key=lambda t: _quality_tuple(t, group_contexts[t.path]), reverse=True) + winner = quality_sorted[0] + losers = [t.path for t in quality_sorted[1:]] + runner_up = quality_sorted[1] if len(quality_sorted) > 1 else None + + metadata_source = _select_metadata_source(group_tracks, group_contexts) + planned_tags, tag_source, tag_source_reason, tag_source_evidence = _build_planned_tags( + winner, metadata_source, group_contexts + ) + group_review: List[str] = list(release_notes) + if not metadata_source: + review_flags.append(f"Missing metadata source for group containing {winner.path}.") + group_review.append("Metadata source missing; tags may be incomplete.") + + meta_changes = _metadata_changes(winner, planned_tags) + winner_context = group_contexts.get(winner.path, "unknown") + winner_quality = _quality_rationale(winner, runner_up, winner_context) + + distance_samples: List[float] = [] + pair_distances: Dict[str, Dict[str, float]] = {t.path: {} for t in group_tracks} + missing_fingerprints = any(not t.fingerprint for t in group_tracks) + for idx, t in enumerate(group_tracks): + for other in group_tracks[idx + 1 :]: + dist = fingerprint_distance(t.fingerprint, other.fingerprint) + distance_samples.append(dist) + pair_distances[t.path][other.path] = dist + pair_distances[other.path][t.path] = dist + max_distance = max(distance_samples) if distance_samples else (1.0 if missing_fingerprints else 0.0) + + track_quality: Dict[str, Dict[str, object]] = { + t.path: { + "container": t.container or t.ext.replace(".", "").upper(), + "codec": t.codec, + "bitrate": t.bitrate, + "sample_rate": t.sample_rate, + "bit_depth": t.bit_depth, + "channels": t.channels, + "is_lossless": t.is_lossless, + } + for t in quality_sorted } - for t in quality_sorted - } - single_art_track, single_art_blob, ambiguous_single_art = _select_single_artwork_candidate( - cluster_tracks, contexts - ) - overall_art_track, overall_art_blob, ambiguous_overall_art = _select_overall_artwork_candidate( - cluster_tracks - ) - artwork_actions: List[ArtworkDirective] = [] - chosen_artwork_source: Dict[str, object] = {"path": None, "reason": ""} - artwork_status = "unchanged" - artwork_evidence: List[str] = [] - - if single_art_blob: - chosen_artwork_source = { - "path": single_art_track.path if single_art_track else None, - "hash": single_art_blob.hash, - "size": single_art_blob.size, - "width": single_art_blob.width, - "height": single_art_blob.height, - "context": "single", - "reason": "Best single artwork by resolution/size", - } - artwork_evidence.append( - f"Single artwork selected from {single_art_track.path if single_art_track else 'unknown'} " - f"({single_art_blob.width}x{single_art_blob.height}, {single_art_blob.size} bytes)" + single_art_track, single_art_blob, ambiguous_single_art = _select_single_artwork_candidate( + group_tracks, group_contexts ) - elif overall_art_blob: - chosen_artwork_source = { - "path": overall_art_track.path if overall_art_track else None, - "hash": overall_art_blob.hash, - "size": overall_art_blob.size, - "width": overall_art_blob.width, - "height": overall_art_blob.height, - "context": "fallback", - "reason": "No single artwork found; best available artwork", - } - artwork_evidence.append( - f"No single artwork found; using best available artwork from {overall_art_track.path if overall_art_track else 'unknown'} " - f"({overall_art_blob.width}x{overall_art_blob.height}, {overall_art_blob.size} bytes)" + overall_art_track, overall_art_blob, ambiguous_overall_art = _select_overall_artwork_candidate( + group_tracks ) - else: - chosen_artwork_source["reason"] = "No artwork candidates available" - artwork_status = "none found" - artwork_evidence.append("No readable artwork candidates were discovered.") - - if winner_context == "album" and single_art_blob and single_art_track: - if not winner.cover_hash or winner.cover_hash != single_art_blob.hash: - artwork_status = "apply" - artwork_actions.append( - ArtworkDirective( - source=single_art_track.path, - target=winner.path, - reason="Apply single artwork to album-context winner", - ) + artwork_actions: List[ArtworkDirective] = [] + chosen_artwork_source: Dict[str, object] = {"path": None, "reason": ""} + artwork_status = "unchanged" + artwork_evidence: List[str] = [] + + if single_art_blob: + chosen_artwork_source = { + "path": single_art_track.path if single_art_track else None, + "hash": single_art_blob.hash, + "size": single_art_blob.size, + "width": single_art_blob.width, + "height": single_art_blob.height, + "context": "single", + "reason": "Best single artwork by resolution/size", + } + artwork_evidence.append( + f"Single artwork selected from {single_art_track.path if single_art_track else 'unknown'} " + f"({single_art_blob.width}x{single_art_blob.height}, {single_art_blob.size} bytes)" + ) + elif overall_art_blob: + chosen_artwork_source = { + "path": overall_art_track.path if overall_art_track else None, + "hash": overall_art_blob.hash, + "size": overall_art_blob.size, + "width": overall_art_blob.width, + "height": overall_art_blob.height, + "context": "fallback", + "reason": "No single artwork found; best available artwork", + } + artwork_evidence.append( + f"No single artwork found; using best available artwork from {overall_art_track.path if overall_art_track else 'unknown'} " + f"({overall_art_blob.width}x{overall_art_blob.height}, {overall_art_blob.size} bytes)" ) - artwork_evidence.append("Album-context winner will receive single artwork to preserve single cover.") else: - artwork_status = "unchanged" - chosen_artwork_source["reason"] = "Winner already has selected single artwork" - artwork_evidence.append("Winner already matches selected single artwork hash; no copy needed.") - elif winner_context == "album" and not single_art_blob and overall_art_blob and overall_art_track: - if not winner.cover_hash or winner.cover_hash != overall_art_blob.hash: - artwork_status = "apply" - artwork_actions.append( - ArtworkDirective( - source=overall_art_track.path, - target=winner.path, - reason="Fill missing album artwork from best available source", + chosen_artwork_source["reason"] = "No artwork candidates available" + artwork_status = "none found" + artwork_evidence.append("No readable artwork candidates were discovered.") + + if winner_context == "album" and single_art_blob and single_art_track: + if not winner.cover_hash or winner.cover_hash != single_art_blob.hash: + artwork_status = "apply" + artwork_actions.append( + ArtworkDirective( + source=single_art_track.path, + target=winner.path, + reason="Apply single artwork to album-context winner", + ) ) - ) - artwork_evidence.append("No single artwork available; applying best available artwork to album winner.") + artwork_evidence.append("Album-context winner will receive single artwork to preserve single cover.") + else: + artwork_status = "unchanged" + chosen_artwork_source["reason"] = "Winner already has selected single artwork" + artwork_evidence.append("Winner already matches selected single artwork hash; no copy needed.") + elif winner_context == "album" and not single_art_blob and overall_art_blob and overall_art_track: + if not winner.cover_hash or winner.cover_hash != overall_art_blob.hash: + artwork_status = "apply" + artwork_actions.append( + ArtworkDirective( + source=overall_art_track.path, + target=winner.path, + reason="Fill missing album artwork from best available source", + ) + ) + artwork_evidence.append("No single artwork available; applying best available artwork to album winner.") + else: + artwork_evidence.append("Album winner artwork already matches best available source.") else: - artwork_evidence.append("Album winner artwork already matches best available source.") - else: - if winner_context == "single": - artwork_evidence.append("Single-context winner retains its own artwork; album art will not be pulled.") - chosen_artwork_source["reason"] = chosen_artwork_source.get("reason") or "Single winner keeps artwork" - elif not single_art_blob and not overall_art_blob: - group_review.append("No artwork available to apply.") - - cover_hashes = {t.cover_hash for t in cluster_tracks if t.cover_hash} - ambiguous_art = ambiguous_single_art or ambiguous_overall_art - if len(cover_hashes) > 1: - ambiguous_art = True - artwork_evidence.append("Conflicting embedded artwork hashes between candidates.") - if ambiguous_art: - review_flags.append( - f"Artwork selection ambiguous for group {_stable_group_id([t.path for t in cluster_tracks])}." - ) - if not (single_art_blob or overall_art_blob): - missing_reason = "No artwork candidates available" - if any(t.artwork_error for t in cluster_tracks): - missing_reason = "Artwork present but unreadable" - review_flags.append(f"{missing_reason} for group {_stable_group_id([t.path for t in cluster_tracks])}.") - chosen_artwork_source["reason"] = missing_reason - artwork_status = "none found" - group_review.append(missing_reason) - - required_tag_gaps = [key for key in ("artist", "title") if not planned_tags.get(key)] - if required_tag_gaps: - group_review.append(f"Missing critical tags: {', '.join(sorted(required_tag_gaps))}.") - - group_confidence = "High (identical fingerprint cluster)" - if missing_fingerprints: - group_confidence = "Low (missing fingerprints in cluster)" - group_review.append("One or more tracks missing fingerprints; requires review.") - near_distances = [d for d in distance_samples if d > exact_duplicate_threshold] - if near_distances: - group_confidence = f"Medium (near-duplicate fingerprint distance up to {max_distance:.3f})" - group_review.append( - f"Audio match requires review (max distance {max_distance:.3f}; near-duplicate threshold {near_duplicate_threshold:.3f})." - ) - if max_distance > near_duplicate_threshold: - group_confidence = f"Low (max fingerprint distance {max_distance:.3f})" - if not missing_fingerprints: + if winner_context == "single": + artwork_evidence.append("Single-context winner retains its own artwork; album art will not be pulled.") + chosen_artwork_source["reason"] = ( + chosen_artwork_source.get("reason") or "Single winner keeps artwork" + ) + elif not single_art_blob and not overall_art_blob: + group_review.append("No artwork available to apply.") + + cover_hashes = {t.cover_hash for t in group_tracks if t.cover_hash} + ambiguous_art = ambiguous_single_art or ambiguous_overall_art + if len(cover_hashes) > 1: + ambiguous_art = True + artwork_evidence.append("Conflicting embedded artwork hashes between candidates.") + if ambiguous_art: + review_flags.append( + f"Artwork selection ambiguous for group {_stable_group_id([t.path for t in group_tracks])}." + ) + if not (single_art_blob or overall_art_blob): + missing_reason = "No artwork candidates available" + if any(t.artwork_error for t in group_tracks): + missing_reason = "Artwork present but unreadable" + review_flags.append(f"{missing_reason} for group {_stable_group_id([t.path for t in group_tracks])}.") + chosen_artwork_source["reason"] = missing_reason + artwork_status = "none found" + group_review.append(missing_reason) + + required_tag_gaps = [key for key in ("artist", "title") if not planned_tags.get(key)] + if required_tag_gaps: + group_review.append(f"Missing critical tags: {', '.join(sorted(required_tag_gaps))}.") + + group_confidence = "High (identical fingerprint cluster)" + if missing_fingerprints: + group_confidence = "Low (missing fingerprints in cluster)" + group_review.append("One or more tracks missing fingerprints; requires review.") + near_distances = [d for d in distance_samples if d > exact_duplicate_threshold] + if near_distances: + group_confidence = f"Medium (near-duplicate fingerprint distance up to {max_distance:.3f})" group_review.append( - f"Fingerprint distances exceed near-duplicate threshold ({near_duplicate_threshold:.3f}); grouping may be unsafe." + f"Audio match requires review (max distance {max_distance:.3f}; near-duplicate threshold {near_duplicate_threshold:.3f})." ) + if max_distance > near_duplicate_threshold: + group_confidence = f"Low (max fingerprint distance {max_distance:.3f})" + if not missing_fingerprints: + group_review.append( + f"Fingerprint distances exceed near-duplicate threshold ({near_duplicate_threshold:.3f}); grouping may be unsafe." + ) - dispositions = {loser: "quarantine" for loser in losers} - playlist_map = {loser: winner.path for loser in losers} - - group_id = _stable_group_id([t.path for t in cluster_tracks]) - if ambiguous_art: - group_review.append("Artwork selection requires review.") - if not losers: - group_review.append("No losers to consolidate.") - if any(_has_review_keyword(t) for t in cluster_tracks): - group_review.append("Contains remix/sped-up variant indicators; review recommended.") - placeholders = _placeholder_present(planned_tags) or any( - _placeholder_present(t.current_tags) for t in cluster_tracks - ) - if placeholders: - group_review.append("Placeholder metadata detected; requires review.") - plan_placeholders = True - if winner.metadata_error: - group_review.append(f"Metadata read issue for winner: {winner.metadata_error}") - if any(t.artwork_error for t in cluster_tracks): - group_review.append("Artwork extraction failed for at least one track.") - - playlist_impact = PlaylistImpact(playlists=len(losers), entries=len(losers)) - track_states = {t.path: dict(t.library_state) for t in cluster_tracks} - - artwork_candidates: List[ArtworkCandidate] = [] - for t in cluster_tracks: - artwork_candidates.extend(t.artwork) - - current_tags = {t.path: _merge_tags(_blank_tags(), t.current_tags) for t in cluster_tracks} - - plans.append( - GroupPlan( - group_id=group_id, - winner_path=winner.path, - losers=losers, - planned_winner_tags=planned_tags, - winner_current_tags=_merge_tags(_blank_tags(), winner.current_tags), - current_tags=current_tags, - metadata_changes=meta_changes, - winner_quality=winner_quality, - artwork=artwork_actions, - artwork_candidates=artwork_candidates, - chosen_artwork_source=chosen_artwork_source, - artwork_status=artwork_status, - loser_disposition=dispositions, - playlist_rewrites=playlist_map, - playlist_impact=playlist_impact, - review_flags=group_review, - context_summary={ - "album": sorted([t.path for t in cluster_tracks if contexts.get(t.path) == "album"]), - "single": sorted([t.path for t in cluster_tracks if contexts.get(t.path) == "single"]), - "unknown": sorted([t.path for t in cluster_tracks if contexts.get(t.path) == "unknown"]), - }, - context_evidence=context_evidence, - tag_source=tag_source, - placeholders_present=placeholders, - tag_source_reason=tag_source_reason, - tag_source_evidence=tag_source_evidence, - track_quality=track_quality, - group_confidence=group_confidence, - group_match_type="Exact" if max_distance <= exact_duplicate_threshold else "Near-duplicate", - grouping_metadata_key=cluster.metadata_key, - grouping_thresholds={ - "exact": cluster.exact_threshold, - "near": cluster.near_threshold, - }, - grouping_decisions=cluster.decisions, - artwork_evidence=artwork_evidence, - fingerprint_distances=pair_distances, - library_state=track_states, + dispositions = {loser: "quarantine" for loser in losers} + playlist_map = {loser: winner.path for loser in losers} + + group_id = _stable_group_id([t.path for t in group_tracks]) + if ambiguous_art: + group_review.append("Artwork selection requires review.") + if not losers: + group_review.append("No losers to consolidate.") + if any(_has_review_keyword(t) for t in group_tracks): + group_review.append("Contains remix/sped-up variant indicators; review recommended.") + placeholders = _placeholder_present(planned_tags) or any( + _placeholder_present(t.current_tags) for t in group_tracks + ) + if placeholders: + group_review.append("Placeholder metadata detected; requires review.") + plan_placeholders = True + if winner.metadata_error: + group_review.append(f"Metadata read issue for winner: {winner.metadata_error}") + if any(t.artwork_error for t in group_tracks): + group_review.append("Artwork extraction failed for at least one track.") + + playlist_impact = PlaylistImpact(playlists=len(losers), entries=len(losers)) + track_states = {t.path: dict(t.library_state) for t in group_tracks} + + artwork_candidates: List[ArtworkCandidate] = [] + for t in group_tracks: + artwork_candidates.extend(t.artwork) + + current_tags = {t.path: _merge_tags(_blank_tags(), t.current_tags) for t in group_tracks} + + plans.append( + GroupPlan( + group_id=group_id, + winner_path=winner.path, + losers=losers, + planned_winner_tags=planned_tags, + winner_current_tags=_merge_tags(_blank_tags(), winner.current_tags), + current_tags=current_tags, + metadata_changes=meta_changes, + winner_quality=winner_quality, + artwork=artwork_actions, + artwork_candidates=artwork_candidates, + chosen_artwork_source=chosen_artwork_source, + artwork_status=artwork_status, + loser_disposition=dispositions, + playlist_rewrites=playlist_map, + playlist_impact=playlist_impact, + review_flags=group_review, + context_summary={ + "album": sorted([p for p in group_paths if group_contexts.get(p) == "album"]), + "single": sorted([p for p in group_paths if group_contexts.get(p) == "single"]), + "unknown": sorted([p for p in group_paths if group_contexts.get(p) == "unknown"]), + }, + context_evidence=group_context_evidence, + tag_source=tag_source, + placeholders_present=placeholders, + tag_source_reason=tag_source_reason, + tag_source_evidence=tag_source_evidence, + track_quality=track_quality, + group_confidence=group_confidence, + group_match_type="Exact" if max_distance <= exact_duplicate_threshold else "Near-duplicate", + grouping_metadata_key=cluster.metadata_key, + grouping_thresholds={ + "exact": cluster.exact_threshold, + "near": cluster.near_threshold, + }, + grouping_decisions=group_decisions, + artwork_evidence=artwork_evidence, + fingerprint_distances=pair_distances, + library_state=track_states, + ) ) - ) if plan_placeholders and "Placeholder metadata detected; review required." not in review_flags: review_flags.append("Placeholder metadata detected; review required.") From 40cee1581c94d18c81502449824b5fa3aa57621c Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Fri, 26 Dec 2025 09:25:03 -0500 Subject: [PATCH 162/606] Wire duplicate finder execute to consolidation runner --- duplicate_consolidation_executor.py | 12 +++++ main_gui.py | 76 ++++++++++++++++++++++++++--- 2 files changed, 81 insertions(+), 7 deletions(-) diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index b67b61b..d798168 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -59,6 +59,7 @@ class ExecutionConfig: allow_deletion: bool = False confirm_deletion: bool = False dry_run_execute: bool = False + retain_losers: bool = False @dataclass @@ -746,6 +747,17 @@ def _find_loser_references( # Step 6: quarantine or delete losers for loser, disposition in loser_disposition.items(): _check_cancel("loser_cleanup") + if config.retain_losers: + quarantine_index[loser] = "retained" + _record( + "loser_cleanup", + loser, + "skipped", + "Loser retained; quarantine disabled.", + disposition=disposition, + group_id=path_to_group.get(loser), + ) + continue if config.dry_run_execute: quarantine_index[loser] = "retained" _record( diff --git a/main_gui.py b/main_gui.py index 377098d..29bf980 100644 --- a/main_gui.py +++ b/main_gui.py @@ -46,6 +46,7 @@ render_consolidation_preview, export_consolidation_preview, ) +from duplicate_consolidation_executor import ExecutionConfig, execute_consolidation_plan from controllers.library_index_controller import generate_index from gui.audio_preview import PlaybackError, VlcPreviewPlayer from io import BytesIO @@ -1451,7 +1452,7 @@ def update_progress(self, value): class DuplicateFinderShell(tk.Toplevel): - """GUI-only shell for the refreshed Duplicate Finder workflow.""" + """GUI shell for the refreshed Duplicate Finder workflow.""" def __init__(self, parent: tk.Widget, library_path: str): super().__init__(parent) @@ -1482,7 +1483,10 @@ def __init__(self, parent: tk.Widget, library_path: str): ).pack(anchor="w") ttk.Label( container, - text="UI shell only – actions are stubbed and do not change your library.", + text=( + "Generate a preview of duplicate groups and execute the consolidation plan. " + "Execution writes backups and reports under the library Docs folder." + ), foreground="#555", wraplength=520, ).pack(anchor="w", pady=(0, 8)) @@ -1592,7 +1596,7 @@ def __init__(self, parent: tk.Widget, library_path: str): self.group_details = ScrolledText(inspector, height=12, state="disabled", wrap="word") self.group_details.pack(fill="both", expand=True, padx=8, pady=8) - self._log_action("Duplicate Finder shell initialized") + self._log_action("Duplicate Finder initialized") def _default_playlist_folder(self, library_path: str) -> str: if library_path: @@ -1827,8 +1831,66 @@ def _handle_execute(self) -> None: self._log_action("Execute blocked: review required groups pending") return - self._log_action("Execute clicked (no mutations; preview reused)") - self._set_status("Executed", progress=100) + playlists_dir = self.playlist_path_var.get().strip() + if not self.update_playlists_var.get(): + playlists_dir = "" + self._log_action("Playlist updates disabled; playlists will not be rewritten.") + elif playlists_dir and not os.path.isdir(playlists_dir): + messagebox.showwarning( + "Playlist Folder Missing", + f"The selected playlist folder does not exist:\n{playlists_dir}", + ) + self._log_action(f"Execute blocked: playlist folder missing ({playlists_dir})") + return + + docs_dir = os.path.join(path, "Docs") + os.makedirs(docs_dir, exist_ok=True) + reports_dir = os.path.join(docs_dir, "duplicate_execution_reports") + + def log_callback(msg: str) -> None: + self.after(0, self._log_action, msg) + + config = ExecutionConfig( + library_root=path, + reports_dir=reports_dir, + playlists_dir=playlists_dir, + quarantine_dir=os.path.join(path, "Quarantine"), + log_callback=log_callback, + allow_review_required=self.override_review_var.get(), + retain_losers=not self.quarantine_var.get(), + ) + + if not self.quarantine_var.get(): + self._log_action("Quarantine disabled; duplicates will be retained in place.") + + self._set_status("Executing…", progress=10) + self._log_action("Execute started") + + def finish(result, error: Exception | None = None) -> None: + if error is not None: + self._log_action(f"Execution failed: {error}") + self._set_status("Execution failed", progress=100) + messagebox.showerror("Execution Failed", str(error)) + return + status = "Executed" if result.success else "Execution failed" + self._set_status(status, progress=100) + self._log_action(f"Execution complete: {'success' if result.success else 'failed'}") + self._log_action(f"Execution report: {result.report_paths.get('html_report')}") + if not result.success: + messagebox.showwarning( + "Execution Failed", + "Execution completed with errors. Review the report for details.", + ) + + def worker() -> None: + try: + result = execute_consolidation_plan(self._plan, config) + except Exception as exc: + self.after(0, finish, None, exc) + return + self.after(0, finish, result, None) + + threading.Thread(target=worker, daemon=True).start() def _toggle_update_playlists(self) -> None: state = "enabled" if self.update_playlists_var.get() else "disabled" @@ -2323,8 +2385,8 @@ def build_ui(self): ttk.Label( df_container, text=( - "Preview the refreshed Duplicate Finder workflow. " - "This shell is UI-only for now and does not modify your library." + "Preview duplicate groups and launch the Duplicate Finder workflow. " + "Execution writes backups and reports under the library Docs folder." ), wraplength=520, justify="left", From 374a5b537b988213e25bf3ea767854e7fe734b89 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Fri, 26 Dec 2025 21:51:08 -0500 Subject: [PATCH 163/606] Show duplicate execution report path --- main_gui.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/main_gui.py b/main_gui.py index 29bf980..23aee38 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1875,11 +1875,16 @@ def finish(result, error: Exception | None = None) -> None: status = "Executed" if result.success else "Execution failed" self._set_status(status, progress=100) self._log_action(f"Execution complete: {'success' if result.success else 'failed'}") - self._log_action(f"Execution report: {result.report_paths.get('html_report')}") + report_path = result.report_paths.get("html_report") + self._log_action(f"Execution report: {report_path}") if not result.success: + report_line = ( + f"\n\nReport (HTML): {report_path}" if report_path else "" + ) messagebox.showwarning( "Execution Failed", - "Execution completed with errors. Review the report for details.", + "Execution completed with errors. Review the report for details." + f"{report_line}", ) def worker() -> None: From 676a255ab3fa724f6d9e40d2d9ad88d39bc909a4 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Fri, 26 Dec 2025 22:02:23 -0500 Subject: [PATCH 164/606] Harden duplicate execution report writing --- duplicate_consolidation_executor.py | 14 +++++++++----- main_gui.py | 15 +++++++++++++-- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index d798168..81b289f 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -124,13 +124,15 @@ def _timestamped_dir(base: str) -> str: def _atomic_write_text(path: str, content: str, *, encoding: str = "utf-8") -> None: directory = os.path.dirname(path) or "." - os.makedirs(directory, exist_ok=True) - tmp = tempfile.NamedTemporaryFile(delete=False, dir=directory) + safe_directory = ensure_long_path(directory) + safe_path = ensure_long_path(path) + os.makedirs(safe_directory, exist_ok=True) + tmp = tempfile.NamedTemporaryFile(delete=False, dir=safe_directory) tmp_path = tmp.name try: with open(tmp_path, "w", encoding=encoding) as handle: handle.write(content) - os.replace(tmp_path, path) + os.replace(ensure_long_path(tmp_path), safe_path) finally: if os.path.exists(tmp_path): try: @@ -919,8 +921,10 @@ def _find_loser_references( html_lines.append("
    ") html_lines.append("") _atomic_write_text(html_report_path, "\n".join(html_lines)) - except Exception: - pass + if not os.path.exists(ensure_long_path(html_report_path)): + log(f"Report write failed: HTML report not found at {html_report_path}") + except Exception as exc: + log(f"Report write failed: unable to create HTML report at {html_report_path} ({exc})") report_paths = { "audit": audit_path, diff --git a/main_gui.py b/main_gui.py index 29bf980..b682e8c 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1875,11 +1875,22 @@ def finish(result, error: Exception | None = None) -> None: status = "Executed" if result.success else "Execution failed" self._set_status(status, progress=100) self._log_action(f"Execution complete: {'success' if result.success else 'failed'}") - self._log_action(f"Execution report: {result.report_paths.get('html_report')}") + report_path = result.report_paths.get("html_report") + self._log_action(f"Execution report: {report_path}") if not result.success: + report_line = "" + if report_path: + if os.path.exists(report_path): + report_line = f"\n\nReport (HTML): {report_path}" + else: + report_line = ( + "\n\nReport (HTML) was not found at the expected path:\n" + f"{report_path}" + ) messagebox.showwarning( "Execution Failed", - "Execution completed with errors. Review the report for details.", + "Execution completed with errors. Review the report for details." + f"{report_line}", ) def worker() -> None: From a7defe7b50020c3e1bd959707b2e83e991d20e59 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Fri, 26 Dec 2025 22:02:35 -0500 Subject: [PATCH 165/606] Revert "Show duplicate execution report path in failure warning" --- main_gui.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/main_gui.py b/main_gui.py index 23aee38..29bf980 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1875,16 +1875,11 @@ def finish(result, error: Exception | None = None) -> None: status = "Executed" if result.success else "Execution failed" self._set_status(status, progress=100) self._log_action(f"Execution complete: {'success' if result.success else 'failed'}") - report_path = result.report_paths.get("html_report") - self._log_action(f"Execution report: {report_path}") + self._log_action(f"Execution report: {result.report_paths.get('html_report')}") if not result.success: - report_line = ( - f"\n\nReport (HTML): {report_path}" if report_path else "" - ) messagebox.showwarning( "Execution Failed", - "Execution completed with errors. Review the report for details." - f"{report_line}", + "Execution completed with errors. Review the report for details.", ) def worker() -> None: From 674cd81075ac8f002f04a2d90f2b0dd0bd80ef1d Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 28 Dec 2025 10:27:17 -0500 Subject: [PATCH 166/606] Record execution errors in audit log --- duplicate_consolidation_executor.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index 81b289f..dd34ad5 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -819,8 +819,10 @@ def _find_loser_references( raise except IndexCancelled: success = False - except Exception: + _record("execution", "execution", "cancelled", "Execution cancelled by user request.") + except Exception as exc: success = False + _record("execution", "execution", "failed", f"Execution failed: {exc}") finally: audit_path = os.path.join(reports_dir, "audit.json") playlist_report_path = os.path.join(reports_dir, "playlist_report.json") From 9b6712895ce1960d1a58a1f20db09786b20ebffa Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 28 Dec 2025 10:44:33 -0500 Subject: [PATCH 167/606] Validate consolidation plans before execution --- duplicate_consolidation.py | 172 ++++++++++++++++++++++++++++ duplicate_consolidation_executor.py | 94 +++++++++++---- 2 files changed, 242 insertions(+), 24 deletions(-) diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index 23b292c..2dc523b 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -788,6 +788,178 @@ def __post_init__(self) -> None: self.plan_signature = hashlib.sha256(canonical.encode("utf-8")).hexdigest() +def _expect_mapping(value: object, field: str) -> Mapping[str, object]: + if not isinstance(value, Mapping): + raise ValueError(f"{field} must be a mapping.") + return value + + +def _expect_list(value: object, field: str) -> List[object]: + if not isinstance(value, list): + raise ValueError(f"{field} must be a list.") + return value + + +def _expect_str(value: object, field: str) -> str: + if not isinstance(value, str): + raise ValueError(f"{field} must be a string.") + return value + + +def _expect_bool(value: object, field: str) -> bool: + if not isinstance(value, bool): + raise ValueError(f"{field} must be a boolean.") + return value + + +def _expect_optional_str(value: object, field: str) -> Optional[str]: + if value is None: + return None + if not isinstance(value, str): + raise ValueError(f"{field} must be a string or null.") + return value + + +def _expect_datetime(value: object, field: str) -> datetime.datetime: + if not isinstance(value, str): + raise ValueError(f"{field} must be an ISO-8601 datetime string.") + try: + return datetime.datetime.fromisoformat(value) + except ValueError as exc: + raise ValueError(f"{field} must be an ISO-8601 datetime string.") from exc + + +def _expect_str_list(value: object, field: str) -> List[str]: + items = _expect_list(value, field) + out: List[str] = [] + for item in items: + if not isinstance(item, str): + raise ValueError(f"{field} must contain strings.") + out.append(item) + return out + + +def _expect_str_mapping(value: object, field: str) -> Dict[str, object]: + mapping = _expect_mapping(value, field) + return {str(k): v for k, v in mapping.items()} + + +def _artwork_candidate_from_dict(raw: Mapping[str, object]) -> ArtworkCandidate: + return ArtworkCandidate( + path=_expect_str(raw.get("path"), "artwork_candidates.path"), + hash=_expect_str(raw.get("hash"), "artwork_candidates.hash"), + size=int(raw.get("size") or 0), + width=raw.get("width") if raw.get("width") is None else int(raw.get("width") or 0), + height=raw.get("height") if raw.get("height") is None else int(raw.get("height") or 0), + status=_expect_str(raw.get("status"), "artwork_candidates.status"), + ) + + +def _artwork_directive_from_dict(raw: Mapping[str, object]) -> ArtworkDirective: + return ArtworkDirective( + source=_expect_str(raw.get("source"), "artwork.source"), + target=_expect_str(raw.get("target"), "artwork.target"), + reason=_expect_str(raw.get("reason"), "artwork.reason"), + ) + + +def _playlist_impact_from_dict(raw: Mapping[str, object]) -> PlaylistImpact: + return PlaylistImpact( + playlists=int(raw.get("playlists") or 0), + entries=int(raw.get("entries") or 0), + ) + + +def _grouping_decision_from_dict(raw: Mapping[str, object]) -> GroupingDecision: + metadata_key = raw.get("metadata_key") + if not isinstance(metadata_key, (list, tuple)) or len(metadata_key) != 2: + raise ValueError("grouping_decisions.metadata_key must be a 2-item list.") + return GroupingDecision( + anchor_path=_expect_str(raw.get("anchor_path"), "grouping_decisions.anchor_path"), + candidate_path=_expect_str(raw.get("candidate_path"), "grouping_decisions.candidate_path"), + metadata_key=(str(metadata_key[0]), str(metadata_key[1])), + coarse_keys_anchor=_expect_str_list(raw.get("coarse_keys_anchor"), "grouping_decisions.coarse_keys_anchor"), + coarse_keys_candidate=_expect_str_list(raw.get("coarse_keys_candidate"), "grouping_decisions.coarse_keys_candidate"), + shared_coarse_keys=_expect_str_list(raw.get("shared_coarse_keys"), "grouping_decisions.shared_coarse_keys"), + distance_to_anchor=float(raw.get("distance_to_anchor") or 0.0), + max_group_distance=float(raw.get("max_group_distance") or 0.0), + threshold=float(raw.get("threshold") or 0.0), + match_type=_expect_str(raw.get("match_type"), "grouping_decisions.match_type"), + coarse_gate=_expect_str(raw.get("coarse_gate"), "grouping_decisions.coarse_gate"), + ) + + +def _group_plan_from_dict(raw: Mapping[str, object]) -> GroupPlan: + metadata_key = raw.get("grouping_metadata_key") + if not isinstance(metadata_key, (list, tuple)) or len(metadata_key) != 2: + raise ValueError("grouping_metadata_key must be a 2-item list.") + + artwork_raw = _expect_list(raw.get("artwork"), "artwork") + artwork_candidates_raw = _expect_list(raw.get("artwork_candidates"), "artwork_candidates") + decisions_raw = _expect_list(raw.get("grouping_decisions"), "grouping_decisions") + playlist_impact_raw = _expect_mapping(raw.get("playlist_impact"), "playlist_impact") + + return GroupPlan( + group_id=_expect_str(raw.get("group_id"), "group_id"), + winner_path=_expect_str(raw.get("winner_path"), "winner_path"), + losers=_expect_str_list(raw.get("losers"), "losers"), + planned_winner_tags=_expect_str_mapping(raw.get("planned_winner_tags"), "planned_winner_tags"), + winner_current_tags=_expect_str_mapping(raw.get("winner_current_tags"), "winner_current_tags"), + current_tags=_expect_mapping(raw.get("current_tags"), "current_tags"), + metadata_changes=_expect_mapping(raw.get("metadata_changes"), "metadata_changes"), + winner_quality=_expect_mapping(raw.get("winner_quality"), "winner_quality"), + artwork=[_artwork_directive_from_dict(_expect_mapping(item, "artwork[]")) for item in artwork_raw], + artwork_candidates=[ + _artwork_candidate_from_dict(_expect_mapping(item, "artwork_candidates[]")) for item in artwork_candidates_raw + ], + chosen_artwork_source=_expect_mapping(raw.get("chosen_artwork_source"), "chosen_artwork_source"), + artwork_status=_expect_str(raw.get("artwork_status"), "artwork_status"), + loser_disposition=_expect_str_mapping(raw.get("loser_disposition"), "loser_disposition"), + playlist_rewrites=_expect_str_mapping(raw.get("playlist_rewrites"), "playlist_rewrites"), + playlist_impact=_playlist_impact_from_dict(playlist_impact_raw), + review_flags=_expect_str_list(raw.get("review_flags"), "review_flags"), + context_summary=_expect_mapping(raw.get("context_summary"), "context_summary"), + context_evidence=_expect_mapping(raw.get("context_evidence"), "context_evidence"), + tag_source=_expect_optional_str(raw.get("tag_source"), "tag_source"), + placeholders_present=_expect_bool(raw.get("placeholders_present"), "placeholders_present"), + tag_source_reason=_expect_str(raw.get("tag_source_reason"), "tag_source_reason"), + tag_source_evidence=_expect_str_list(raw.get("tag_source_evidence"), "tag_source_evidence"), + track_quality=_expect_mapping(raw.get("track_quality"), "track_quality"), + group_confidence=_expect_str(raw.get("group_confidence"), "group_confidence"), + group_match_type=_expect_str(raw.get("group_match_type"), "group_match_type"), + grouping_metadata_key=(str(metadata_key[0]), str(metadata_key[1])), + grouping_thresholds=_expect_mapping(raw.get("grouping_thresholds"), "grouping_thresholds"), + grouping_decisions=[ + _grouping_decision_from_dict(_expect_mapping(item, "grouping_decisions[]")) for item in decisions_raw + ], + artwork_evidence=_expect_str_list(raw.get("artwork_evidence"), "artwork_evidence"), + fingerprint_distances=_expect_mapping(raw.get("fingerprint_distances"), "fingerprint_distances"), + library_state=_expect_mapping(raw.get("library_state"), "library_state"), + ) + + +def consolidation_plan_from_dict(raw: Mapping[str, object]) -> ConsolidationPlan: + """Convert a serialized plan mapping into a ConsolidationPlan.""" + groups_raw = _expect_list(raw.get("groups"), "groups") + review_flags = _expect_str_list(raw.get("review_flags"), "review_flags") + generated_at = _expect_datetime(raw.get("generated_at"), "generated_at") + placeholders_present = _expect_bool(raw.get("placeholders_present"), "placeholders_present") + source_snapshot = _expect_mapping(raw.get("source_snapshot"), "source_snapshot") + plan_signature = raw.get("plan_signature") + if plan_signature is not None and not isinstance(plan_signature, str): + raise ValueError("plan_signature must be a string or null.") + + groups = [_group_plan_from_dict(_expect_mapping(item, "groups[]")) for item in groups_raw] + return ConsolidationPlan( + groups=groups, + review_flags=review_flags, + generated_at=generated_at, + placeholders_present=placeholders_present, + source_snapshot=source_snapshot, + plan_signature=plan_signature, + ) + + def _normalize_track(raw: Mapping[str, object]) -> DuplicateTrack: path = str(raw.get("path")) ext = os.path.splitext(path)[1].lower() or str(raw.get("ext", "")).lower() diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index dd34ad5..7bb9fcc 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -23,7 +23,12 @@ from dataclasses import dataclass, field from typing import Callable, Dict, Iterable, List, Mapping, MutableMapping, Sequence -from duplicate_consolidation import ArtworkDirective, ConsolidationPlan, _capture_library_state +from duplicate_consolidation import ( + ArtworkDirective, + ConsolidationPlan, + _capture_library_state, + consolidation_plan_from_dict, +) from indexer_control import IndexCancelled, cancel_event as global_cancel_event from utils.path_helpers import ensure_long_path @@ -396,7 +401,35 @@ def _detect_state_drift(expected: Mapping[str, object], actual: Mapping[str, obj return None -def execute_consolidation_plan(plan: ConsolidationPlan, config: ExecutionConfig) -> ExecutionResult: +def _coerce_consolidation_plan( + plan_input: ConsolidationPlan | Mapping[str, object] | str, + log: Callable[[str], None], +) -> tuple[ConsolidationPlan, str]: + if isinstance(plan_input, ConsolidationPlan): + return plan_input, "in-memory plan" + payload: object = plan_input + source = "plan payload" + if isinstance(plan_input, str): + try: + payload = json.loads(plan_input) + except json.JSONDecodeError as exc: + raise ValueError(f"Plan JSON could not be decoded: {exc}") from exc + source = "json payload" + if isinstance(payload, Mapping): + if "plan" in payload: + inner = payload.get("plan") + if not isinstance(inner, Mapping): + raise ValueError("Plan payload has a non-mapping 'plan' field.") + payload = inner + source = f"{source} (wrapped)" + return consolidation_plan_from_dict(payload), source + raise ValueError("Plan input must be a ConsolidationPlan, mapping, or JSON string.") + + +def execute_consolidation_plan( + plan_input: ConsolidationPlan | Mapping[str, object] | str, + config: ExecutionConfig, +) -> ExecutionResult: """Execute the provided consolidation plan with safety guarantees.""" cancel = config.cancel_event or global_cancel_event @@ -404,6 +437,9 @@ def execute_consolidation_plan(plan: ConsolidationPlan, config: ExecutionConfig) actions: List[ExecutionAction] = [] playlist_results: List[PlaylistRewriteResult] = [] quarantine_index: Dict[str, str] = {} + group_lookup: Dict[str, object] = {} + plan: ConsolidationPlan | None = None + computed_signature = "" run_root = _timestamped_dir(config.reports_dir) reports_dir = os.path.join(run_root, "reports") @@ -415,24 +451,6 @@ def execute_consolidation_plan(plan: ConsolidationPlan, config: ExecutionConfig) playlists_dir = config.playlists_dir or os.path.join(config.library_root, "Playlists") quarantine_dir = config.quarantine_dir or os.path.join(config.library_root, "Quarantine") - loser_to_winner: Dict[str, str] = {} - loser_disposition: Dict[str, str] = {} - artwork_actions: List[ArtworkDirective] = [] - planned_tags: Dict[str, Dict[str, object]] = {} - path_to_group: Dict[str, str] = {} - group_lookup: Dict[str, object] = {g.group_id: g for g in plan.groups} - playlist_targets: Dict[str, str] = {} - playlist_base_dirs: Dict[str, str] = {} - - for group in plan.groups: - planned_tags[group.winner_path] = dict(group.planned_winner_tags) - for loser in group.losers: - loser_to_winner[loser] = group.winner_path - loser_disposition[loser] = group.loser_disposition.get(loser, "quarantine") - path_to_group[loser] = group.group_id - path_to_group[group.winner_path] = group.group_id - artwork_actions.extend(group.artwork) - def _record(step: str, target: str, status: str, detail: str, **metadata: object) -> None: actions.append(ExecutionAction(step=step, target=target, status=status, detail=detail, metadata=metadata)) @@ -441,6 +459,14 @@ def _check_cancel(stage: str) -> None: _record(stage, "execution", "cancelled", "Execution cancelled by user request.") raise IndexCancelled() + loser_to_winner: Dict[str, str] = {} + loser_disposition: Dict[str, str] = {} + artwork_actions: List[ArtworkDirective] = [] + planned_tags: Dict[str, Dict[str, object]] = {} + path_to_group: Dict[str, str] = {} + playlist_targets: Dict[str, str] = {} + playlist_base_dirs: Dict[str, str] = {} + def _find_loser_references( working_playlists: Mapping[str, str], playlist_base_dirs: Mapping[str, str] | None, @@ -460,9 +486,27 @@ def _find_loser_references( return refs success = True - computed_signature = _compute_plan_signature(plan) try: + try: + plan, plan_source = _coerce_consolidation_plan(plan_input, log) + log(f"Execute plan input resolved from {plan_source}.") + except ValueError as exc: + success = False + _record("preflight", "plan", "blocked", f"Invalid consolidation plan: {exc}") + raise RuntimeError(f"Execution blocked: invalid consolidation plan ({exc})") from exc + + group_lookup = {g.group_id: g for g in plan.groups} + for group in plan.groups: + planned_tags[group.winner_path] = dict(group.planned_winner_tags) + for loser in group.losers: + loser_to_winner[loser] = group.winner_path + loser_disposition[loser] = group.loser_disposition.get(loser, "quarantine") + path_to_group[loser] = group.group_id + path_to_group[group.winner_path] = group.group_id + artwork_actions.extend(group.artwork) + + computed_signature = _compute_plan_signature(plan) _check_cancel("start") if plan.plan_signature and plan.plan_signature != computed_signature: @@ -829,13 +873,15 @@ def _find_loser_references( quarantine_index_path = os.path.join(reports_dir, "quarantine_index.json") html_report_path = os.path.join(reports_dir, "execution_report.html") + plan_signature = plan.plan_signature if plan else None + source_snapshot = plan.source_snapshot if plan else {} audit_payload = { "success": success, "actions": [a.to_dict() for a in actions], "generated_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), - "plan_signature": plan.plan_signature or computed_signature, + "plan_signature": plan_signature or computed_signature, "computed_signature": computed_signature, - "source_snapshot": {k: dict(v) for k, v in plan.source_snapshot.items()}, + "source_snapshot": {k: dict(v) for k, v in source_snapshot.items()}, } playlist_payload = { "backups_dir": backups_dir, @@ -879,7 +925,7 @@ def _find_loser_references( "", "

    Execution Report

    ", f"

    Status: {'success' if success else 'failed'}

    ", - f"

    Plan signature: {plan.plan_signature or computed_signature}

    ", + f"

    Plan signature: {plan_signature or computed_signature}

    ", f"

    Reports directory: {reports_dir}

    ", ] From 933458d28fe83d4b10f5fd9d4a61252b6a488919 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 28 Dec 2025 11:11:48 -0500 Subject: [PATCH 168/606] Fix duplicate finder execution report path --- duplicate_consolidation_executor.py | 10 ++++++++- main_gui.py | 32 ++++++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index 7bb9fcc..ba41788 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -151,6 +151,12 @@ def _atomic_write_json(path: str, payload: Mapping[str, object]) -> None: _atomic_write_text(path, text) +def _ensure_html_extension(path: str) -> str: + if path.lower().endswith(".html"): + return path + return f"{path}.html" + + def _iter_playlists(playlists_dir: str) -> Iterable[str]: if not os.path.isdir(playlists_dir): return [] @@ -871,7 +877,9 @@ def _find_loser_references( audit_path = os.path.join(reports_dir, "audit.json") playlist_report_path = os.path.join(reports_dir, "playlist_report.json") quarantine_index_path = os.path.join(reports_dir, "quarantine_index.json") - html_report_path = os.path.join(reports_dir, "execution_report.html") + html_report_path = _ensure_html_extension( + os.path.join(reports_dir, "execution_report") + ) plan_signature = plan.plan_signature if plan else None source_snapshot = plan.source_snapshot if plan else {} diff --git a/main_gui.py b/main_gui.py index b682e8c..c2272cf 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1471,6 +1471,7 @@ def __init__(self, parent: tk.Widget, library_path: str): self.override_review_var = tk.BooleanVar(value=False) self.preview_html_path: str | None = None self.preview_json_path: str | None = None + self.execution_report_path: str | None = None self._plan = None container = ttk.Frame(self) @@ -1528,6 +1529,10 @@ def __init__(self, parent: tk.Widget, library_path: str): controls, text="Open Preview", command=self._open_preview_output, state="disabled" ) self.open_preview_btn.pack(side="left", padx=(0, 6)) + self.open_report_btn = ttk.Button( + controls, text="Open Report", command=self._open_execution_report, state="disabled" + ) + self.open_report_btn.pack(side="left", padx=(0, 6)) ttk.Button(controls, text="Execute", command=self._handle_execute).pack( side="left", padx=(0, 12) ) @@ -1875,8 +1880,14 @@ def finish(result, error: Exception | None = None) -> None: status = "Executed" if result.success else "Execution failed" self._set_status(status, progress=100) self._log_action(f"Execution complete: {'success' if result.success else 'failed'}") - report_path = result.report_paths.get("html_report") + report_path = self._normalize_html_report_path(result.report_paths.get("html_report")) self._log_action(f"Execution report: {report_path}") + if report_path and os.path.exists(report_path): + self.execution_report_path = report_path + self.open_report_btn.config(state="normal") + else: + self.execution_report_path = None + self.open_report_btn.config(state="disabled") if not result.success: report_line = "" if report_path: @@ -1920,6 +1931,25 @@ def _open_preview_output(self) -> None: return webbrowser.open(self.preview_html_path) + def _normalize_html_report_path(self, report_path: str | None) -> str | None: + if not report_path: + return None + if report_path.lower().endswith(".html"): + return report_path + return f"{report_path}.html" + + def _open_execution_report(self) -> None: + if not self.execution_report_path: + messagebox.showinfo("Execution Report", "No execution report is available yet.") + return + if not os.path.exists(self.execution_report_path): + messagebox.showerror( + "Execution Report Missing", + "Execution report could not be found. Run the execution again to generate a new report.", + ) + return + webbrowser.open(self.execution_report_path) + class SoundVaultImporterApp(tk.Tk): def __init__(self): super().__init__() From ce89556ad9fa56ff89727ebabc3951446d259ee7 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 28 Dec 2025 12:00:29 -0500 Subject: [PATCH 169/606] Fix duplicate finder execution report opener --- main_gui.py | 53 +++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/main_gui.py b/main_gui.py index c2272cf..402f48d 100644 --- a/main_gui.py +++ b/main_gui.py @@ -4,6 +4,7 @@ import logging import webbrowser import re +from pathlib import Path if sys.platform == "win32": try: @@ -88,6 +89,7 @@ from playlist_engine import bucket_by_tempo_energy, autodj_playlist from controllers.cluster_controller import gather_tracks from playlist_generator import write_playlist +from utils.path_helpers import ensure_long_path, strip_ext_prefix FilterFn = Callable[[FileRecord], bool] _cached_filters = None @@ -1882,7 +1884,7 @@ def finish(result, error: Exception | None = None) -> None: self._log_action(f"Execution complete: {'success' if result.success else 'failed'}") report_path = self._normalize_html_report_path(result.report_paths.get("html_report")) self._log_action(f"Execution report: {report_path}") - if report_path and os.path.exists(report_path): + if report_path and os.path.exists(ensure_long_path(report_path)): self.execution_report_path = report_path self.open_report_btn.config(state="normal") else: @@ -1923,13 +1925,12 @@ def _toggle_quarantine(self) -> None: self._log_action(f"Quarantine Duplicates {state}") def _open_preview_output(self) -> None: - if not self.preview_html_path: - messagebox.showinfo("Preview", "No preview has been generated yet.") - return - if not os.path.exists(self.preview_html_path): - messagebox.showerror("Preview Missing", "Preview file could not be found. Generate a new preview.") - return - webbrowser.open(self.preview_html_path) + self._open_local_html( + self.preview_html_path, + title="Preview", + missing_message="Preview file could not be found. Generate a new preview.", + empty_message="No preview has been generated yet.", + ) def _normalize_html_report_path(self, report_path: str | None) -> str | None: if not report_path: @@ -1939,16 +1940,36 @@ def _normalize_html_report_path(self, report_path: str | None) -> str | None: return f"{report_path}.html" def _open_execution_report(self) -> None: - if not self.execution_report_path: - messagebox.showinfo("Execution Report", "No execution report is available yet.") + self._open_local_html( + self.execution_report_path, + title="Execution Report", + missing_message=( + "Execution report could not be found. Run the execution again to generate a new report." + ), + empty_message="No execution report is available yet.", + ) + + def _open_local_html( + self, + path: str | None, + *, + title: str, + missing_message: str, + empty_message: str, + ) -> None: + if not path: + messagebox.showinfo(title, empty_message) return - if not os.path.exists(self.execution_report_path): - messagebox.showerror( - "Execution Report Missing", - "Execution report could not be found. Run the execution again to generate a new report.", - ) + safe_path = ensure_long_path(path) + if not os.path.exists(safe_path): + messagebox.showerror(f"{title} Missing", missing_message) return - webbrowser.open(self.execution_report_path) + display_path = strip_ext_prefix(path) + try: + uri = Path(display_path).resolve().as_uri() + except Exception: + uri = display_path + webbrowser.open(uri) class SoundVaultImporterApp(tk.Tk): def __init__(self): From 5ca9dd3a5eafe9fb3b75bfafc52538a45913b017 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 28 Dec 2025 12:31:15 -0500 Subject: [PATCH 170/606] Add delete option to duplicate finder UI --- main_gui.py | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/main_gui.py b/main_gui.py index 402f48d..9c9d12e 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1470,6 +1470,7 @@ def __init__(self, parent: tk.Widget, library_path: str): ) self.update_playlists_var = tk.BooleanVar(value=False) self.quarantine_var = tk.BooleanVar(value=True) + self.delete_losers_var = tk.BooleanVar(value=False) self.override_review_var = tk.BooleanVar(value=False) self.preview_html_path: str | None = None self.preview_json_path: str | None = None @@ -1550,6 +1551,12 @@ def __init__(self, parent: tk.Widget, library_path: str): variable=self.quarantine_var, command=self._toggle_quarantine, ).pack(side="left") + ttk.Checkbutton( + controls, + text="Delete losers (permanent)", + variable=self.delete_losers_var, + command=self._toggle_delete_losers, + ).pack(side="left", padx=(10, 0)) ttk.Checkbutton( controls, text="Override review blocks", @@ -1784,7 +1791,7 @@ def _generate_plan(self, write_preview: bool) -> None: self._set_status("Building plan…", progress=25) plan = build_consolidation_plan(tracks) self._plan = plan - self._update_groups_view(plan) + self._apply_deletion_mode() docs_dir = os.path.join(path, "Docs") os.makedirs(docs_dir, exist_ok=True) @@ -1837,6 +1844,13 @@ def _handle_execute(self) -> None: ) self._log_action("Execute blocked: review required groups pending") return + if self.delete_losers_var.get() and not self.quarantine_var.get(): + messagebox.showwarning( + "Cleanup Required", + "Deletion requires cleanup to be enabled. Check Quarantine Duplicates or disable deletion.", + ) + self._log_action("Execute blocked: deletion enabled while cleanup disabled") + return playlists_dir = self.playlist_path_var.get().strip() if not self.update_playlists_var.get(): @@ -1865,10 +1879,14 @@ def log_callback(msg: str) -> None: log_callback=log_callback, allow_review_required=self.override_review_var.get(), retain_losers=not self.quarantine_var.get(), + allow_deletion=self.delete_losers_var.get(), + confirm_deletion=self.delete_losers_var.get(), ) if not self.quarantine_var.get(): self._log_action("Quarantine disabled; duplicates will be retained in place.") + if self.delete_losers_var.get(): + self._log_action("Deletion enabled; losers will be deleted during execution.") self._set_status("Executing…", progress=10) self._log_action("Execute started") @@ -1923,6 +1941,43 @@ def _toggle_update_playlists(self) -> None: def _toggle_quarantine(self) -> None: state = "enabled" if self.quarantine_var.get() else "disabled" self._log_action(f"Quarantine Duplicates {state}") + if not self.quarantine_var.get() and self.delete_losers_var.get(): + messagebox.showwarning( + "Cleanup Required", + "Deletion requires cleanup to be enabled. Re-enabling Quarantine Duplicates.", + ) + self.quarantine_var.set(True) + self._log_action("Quarantine Duplicates enabled to support deletions.") + self._reset_preview_if_needed() + + def _toggle_delete_losers(self) -> None: + if self.delete_losers_var.get(): + if not self.quarantine_var.get(): + self.quarantine_var.set(True) + self._log_action("Quarantine Duplicates enabled to support deletions.") + self._log_action("Delete losers enabled; preview will mark losers for deletion.") + else: + self._log_action("Delete losers disabled; preview will quarantine losers.") + self._apply_deletion_mode() + self._reset_preview_if_needed() + + def _apply_deletion_mode(self) -> None: + if not self._plan: + return + delete_mode = self.delete_losers_var.get() + for group in self._plan.groups: + for loser in group.losers: + current = group.loser_disposition.get(loser, "quarantine") + if current in ("quarantine", "delete"): + group.loser_disposition[loser] = "delete" if delete_mode else "quarantine" + self._update_groups_view(self._plan) + + def _reset_preview_if_needed(self) -> None: + if self.preview_html_path or self.preview_json_path: + self.preview_html_path = None + self.preview_json_path = None + self.open_preview_btn.config(state="disabled") + self._log_action("Preview cleared; generate a new preview to reflect delete settings.") def _open_preview_output(self) -> None: self._open_local_html( From 47dd271e6d5dd2236d57395c3b69d95d954aa6e0 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 28 Dec 2025 15:31:45 -0500 Subject: [PATCH 171/606] Add per-group duplicate dispositions --- duplicate_consolidation_executor.py | 21 +++-- main_gui.py | 118 +++++++++++++++++++++++++--- 2 files changed, 124 insertions(+), 15 deletions(-) diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index ba41788..4063d2e 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -799,14 +799,20 @@ def _find_loser_references( # Step 6: quarantine or delete losers for loser, disposition in loser_disposition.items(): _check_cancel("loser_cleanup") - if config.retain_losers: + effective = disposition if disposition in ("retain", "quarantine", "delete") else None + if effective is None: + effective = "retain" if config.retain_losers else "quarantine" + if effective == "retain": quarantine_index[loser] = "retained" + detail = "Loser retained by plan disposition." + if config.dry_run_execute: + detail = "Dry-run execute enabled; loser retained in place." _record( "loser_cleanup", loser, "skipped", - "Loser retained; quarantine disabled.", - disposition=disposition, + detail, + disposition=effective, group_id=path_to_group.get(loser), ) continue @@ -817,11 +823,11 @@ def _find_loser_references( loser, "skipped", "Dry-run execute enabled; loser not moved or deleted.", - disposition=disposition, + disposition=effective, group_id=path_to_group.get(loser), ) continue - if disposition == "delete": + if effective == "delete": try: if os.path.exists(loser): os.remove(loser) @@ -831,6 +837,7 @@ def _find_loser_references( loser, "success", "Loser deleted.", + disposition=effective, group_id=path_to_group.get(loser), ) except Exception as exc: @@ -840,6 +847,7 @@ def _find_loser_references( loser, "failed", f"Failed to delete loser: {exc}", + disposition=effective, group_id=path_to_group.get(loser), ) raise @@ -855,6 +863,7 @@ def _find_loser_references( "success", "Loser quarantined.", quarantine_path=dest, + disposition=effective, group_id=path_to_group.get(loser), ) except Exception as exc: @@ -864,6 +873,8 @@ def _find_loser_references( loser, "failed", f"Failed to quarantine loser: {exc}", + quarantine_path=dest, + disposition=effective, group_id=path_to_group.get(loser), ) raise diff --git a/main_gui.py b/main_gui.py index 9c9d12e..8278969 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1472,6 +1472,9 @@ def __init__(self, parent: tk.Widget, library_path: str): self.quarantine_var = tk.BooleanVar(value=True) self.delete_losers_var = tk.BooleanVar(value=False) self.override_review_var = tk.BooleanVar(value=False) + self.group_disposition_var = tk.StringVar(value="") + self.group_disposition_overrides: dict[str, str] = {} + self._selected_group_id: str | None = None self.preview_html_path: str | None = None self.preview_json_path: str | None = None self.execution_report_path: str | None = None @@ -1607,8 +1610,20 @@ def __init__(self, parent: tk.Widget, library_path: str): inspector = ttk.LabelFrame(results, text="Group Details") inspector.grid(row=0, column=1, sticky="nsew", padx=(3, 6), pady=6) + disposition_row = ttk.Frame(inspector) + disposition_row.pack(fill="x", padx=8, pady=(8, 4)) + ttk.Label(disposition_row, text="Group disposition").pack(side="left") + self.group_disposition_menu = ttk.Combobox( + disposition_row, + textvariable=self.group_disposition_var, + state="disabled", + width=22, + values=("Default (global)", "Retain", "Quarantine", "Delete"), + ) + self.group_disposition_menu.pack(side="left", padx=(6, 0)) + self.group_disposition_menu.bind("<>", self._on_group_disposition_change) self.group_details = ScrolledText(inspector, height=12, state="disabled", wrap="word") - self.group_details.pack(fill="both", expand=True, padx=8, pady=8) + self.group_details.pack(fill="both", expand=True, padx=8, pady=(0, 8)) self._log_action("Duplicate Finder initialized") @@ -1628,6 +1643,18 @@ def _log_action(self, message: str) -> None: self.log_text.see("end") self.log_text.configure(state="disabled") + def _default_group_disposition(self) -> str: + if not self.quarantine_var.get(): + return "retain" + if self.delete_losers_var.get(): + return "delete" + return "quarantine" + + def _reset_group_selection(self) -> None: + self._selected_group_id = None + self.group_disposition_var.set("Default (global)") + self.group_disposition_menu.configure(state="disabled") + def _update_groups_view(self, plan) -> None: self.groups_tree.delete(*self.groups_tree.get_children()) for group in plan.groups: @@ -1645,6 +1672,7 @@ def _update_groups_view(self, plan) -> None: self.group_details.delete("1.0", "end") self.group_details.insert("end", "Select a group to view details.") self.group_details.configure(state="disabled") + self._reset_group_selection() def _on_group_select(self, event=None) -> None: sel = self.groups_tree.selection() @@ -1653,16 +1681,31 @@ def _on_group_select(self, event=None) -> None: group_id = sel[0] group = next((g for g in self._plan.groups if g.group_id == group_id), None) if group: + self._selected_group_id = group.group_id + self.group_disposition_menu.configure(state="readonly") + override = self.group_disposition_overrides.get(group.group_id) + if override == "retain": + self.group_disposition_var.set("Retain") + elif override == "quarantine": + self.group_disposition_var.set("Quarantine") + elif override == "delete": + self.group_disposition_var.set("Delete") + else: + self.group_disposition_var.set("Default (global)") self._render_group_details(group) def _render_group_details(self, group) -> None: + default_disp = self._default_group_disposition() + override = self.group_disposition_overrides.get(group.group_id) + disposition_label = f"{override} (override)" if override else f"{default_disp} (default)" lines = [ f"Winner: {group.winner_path}", f"Losers: {len(group.losers)}", + f"Group disposition: {disposition_label}", "Dispositions:", ] for loser in group.losers: - disp = group.loser_disposition.get(loser, "quarantine") + disp = group.loser_disposition.get(loser, default_disp) playlist = group.playlist_rewrites.get(loser, "n/a") lines.append(f" - {loser} → {disp} (playlist → {playlist})") @@ -1691,6 +1734,49 @@ def _render_group_details(self, group) -> None: self.group_details.insert("end", "\n".join(lines)) self.group_details.configure(state="disabled") + def _on_group_disposition_change(self, event=None) -> None: + if not self._plan or not self._selected_group_id: + return + group = next( + (g for g in self._plan.groups if g.group_id == self._selected_group_id), + None, + ) + if not group: + return + selection = self.group_disposition_var.get() + choice_map = { + "Default (global)": None, + "Retain": "retain", + "Quarantine": "quarantine", + "Delete": "delete", + } + disposition = choice_map.get(selection) + if disposition is None: + self.group_disposition_overrides.pop(group.group_id, None) + disposition = self._default_group_disposition() + self._log_action(f"Group {group.group_id} disposition reset to default ({disposition}).") + else: + self.group_disposition_overrides[group.group_id] = disposition + self._log_action(f"Group {group.group_id} disposition override set to {disposition}.") + + for loser in group.losers: + group.loser_disposition[loser] = disposition + + self._render_group_details(group) + self._reset_preview_if_needed() + + def _count_group_dispositions(self) -> dict[str, int]: + counts = {"retain": 0, "quarantine": 0, "delete": 0} + if not self._plan: + return counts + default_disposition = self._default_group_disposition() + for group in self._plan.groups: + for loser in group.losers: + disp = group.loser_disposition.get(loser, default_disposition) + if disp in counts: + counts[disp] += 1 + return counts + def _set_status(self, status: str, progress: float | None = None) -> None: self.status_var.set(status) if progress is not None: @@ -1791,6 +1877,7 @@ def _generate_plan(self, write_preview: bool) -> None: self._set_status("Building plan…", progress=25) plan = build_consolidation_plan(tracks) self._plan = plan + self.group_disposition_overrides.clear() self._apply_deletion_mode() docs_dir = os.path.join(path, "Docs") @@ -1871,6 +1958,10 @@ def _handle_execute(self) -> None: def log_callback(msg: str) -> None: self.after(0, self._log_action, msg) + disposition_counts = self._count_group_dispositions() + deletions_requested = disposition_counts.get("delete", 0) > 0 + quarantines_requested = disposition_counts.get("quarantine", 0) > 0 + config = ExecutionConfig( library_root=path, reports_dir=reports_dir, @@ -1879,14 +1970,19 @@ def log_callback(msg: str) -> None: log_callback=log_callback, allow_review_required=self.override_review_var.get(), retain_losers=not self.quarantine_var.get(), - allow_deletion=self.delete_losers_var.get(), - confirm_deletion=self.delete_losers_var.get(), + allow_deletion=deletions_requested, + confirm_deletion=deletions_requested, ) if not self.quarantine_var.get(): - self._log_action("Quarantine disabled; duplicates will be retained in place.") - if self.delete_losers_var.get(): - self._log_action("Deletion enabled; losers will be deleted during execution.") + if quarantines_requested or deletions_requested: + self._log_action( + "Quarantine disabled globally; group overrides will still move or delete selected losers." + ) + else: + self._log_action("Quarantine disabled; duplicates will be retained in place.") + if deletions_requested: + self._log_action("Deletion enabled; selected losers will be deleted during execution.") self._set_status("Executing…", progress=10) self._log_action("Execute started") @@ -1964,12 +2060,14 @@ def _toggle_delete_losers(self) -> None: def _apply_deletion_mode(self) -> None: if not self._plan: return - delete_mode = self.delete_losers_var.get() + default_disposition = self._default_group_disposition() for group in self._plan.groups: + if group.group_id in self.group_disposition_overrides: + continue for loser in group.losers: current = group.loser_disposition.get(loser, "quarantine") - if current in ("quarantine", "delete"): - group.loser_disposition[loser] = "delete" if delete_mode else "quarantine" + if current in ("retain", "quarantine", "delete"): + group.loser_disposition[loser] = default_disposition self._update_groups_view(self._plan) def _reset_preview_if_needed(self) -> None: From cc7db88b80cbfade8710e1ce1cb0d5aef3a86802 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 28 Dec 2025 16:07:16 -0500 Subject: [PATCH 172/606] Enable execution from preview output --- duplicate_consolidation_executor.py | 22 ++++-- main_gui.py | 36 ++++++++- .../test_duplicate_consolidation_hardening.py | 76 +++++++++++++++++++ 3 files changed, 126 insertions(+), 8 deletions(-) diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index 4063d2e..557f7e7 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -416,11 +416,23 @@ def _coerce_consolidation_plan( payload: object = plan_input source = "plan payload" if isinstance(plan_input, str): - try: - payload = json.loads(plan_input) - except json.JSONDecodeError as exc: - raise ValueError(f"Plan JSON could not be decoded: {exc}") from exc - source = "json payload" + if os.path.exists(plan_input): + if not os.path.isfile(plan_input): + raise ValueError(f"Plan path is not a file: {plan_input}") + try: + with open(plan_input, "r", encoding="utf-8") as handle: + payload = json.load(handle) + except OSError as exc: + raise ValueError(f"Plan file could not be read: {exc}") from exc + except json.JSONDecodeError as exc: + raise ValueError(f"Plan JSON could not be decoded: {exc}") from exc + source = f"preview output {plan_input}" + else: + try: + payload = json.loads(plan_input) + except json.JSONDecodeError as exc: + raise ValueError(f"Plan JSON could not be decoded: {exc}") from exc + source = "json payload" if isinstance(payload, Mapping): if "plan" in payload: inner = payload.get("plan") diff --git a/main_gui.py b/main_gui.py index 8278969..d9f32d0 100644 --- a/main_gui.py +++ b/main_gui.py @@ -44,6 +44,7 @@ ) from duplicate_consolidation import ( build_consolidation_plan, + consolidation_plan_from_dict, render_consolidation_preview, export_consolidation_preview, ) @@ -1905,6 +1906,22 @@ def _generate_plan(self, write_preview: bool) -> None: if plan.review_required_count: self._log_action("Review required groups will block execution unless overridden.") + def _load_preview_plan(self, preview_path: str): + try: + with open(preview_path, "r", encoding="utf-8") as handle: + payload = json.load(handle) + except (OSError, json.JSONDecodeError) as exc: + self._log_action(f"Preview plan could not be loaded: {exc}") + return None + if not isinstance(payload, dict) or "plan" not in payload: + self._log_action("Preview plan could not be loaded: missing plan payload.") + return None + try: + return consolidation_plan_from_dict(payload["plan"]) + except ValueError as exc: + self._log_action(f"Preview plan could not be loaded: {exc}") + return None + def _handle_scan(self) -> None: path = self._validate_library_root() if not path: @@ -1920,11 +1937,24 @@ def _handle_execute(self) -> None: path = self._validate_library_root() if not path: return - if not self._plan: + preview_plan_path = self.preview_json_path or os.path.join(path, "Docs", "duplicate_preview.json") + plan_input = self._plan + plan_for_checks = self._plan + if preview_plan_path and os.path.exists(preview_plan_path): + if not self.preview_json_path: + self.preview_json_path = preview_plan_path + if not plan_for_checks: + plan_for_checks = self._load_preview_plan(preview_plan_path) + if plan_for_checks: + self._plan = plan_for_checks + plan_input = preview_plan_path + self._log_action(f"Using preview output for execution: {preview_plan_path}") + + if not plan_input: messagebox.showwarning("Preview Required", "Generate a preview before executing.") self._log_action("Execute blocked: no plan generated") return - if self._plan.review_required_count and not self.override_review_var.get(): + if plan_for_checks and plan_for_checks.review_required_count and not self.override_review_var.get(): messagebox.showwarning( "Review Required", "Resolve review-required groups or check Override review blocks to proceed.", @@ -2022,7 +2052,7 @@ def finish(result, error: Exception | None = None) -> None: def worker() -> None: try: - result = execute_consolidation_plan(self._plan, config) + result = execute_consolidation_plan(plan_input, config) except Exception as exc: self.after(0, finish, None, exc) return diff --git a/tests/test_duplicate_consolidation_hardening.py b/tests/test_duplicate_consolidation_hardening.py index 8f1823b..5622bd9 100644 --- a/tests/test_duplicate_consolidation_hardening.py +++ b/tests/test_duplicate_consolidation_hardening.py @@ -11,6 +11,7 @@ _extract_artwork_from_audio, _metadata_changes, build_consolidation_plan, + export_consolidation_preview, ) from duplicate_consolidation_executor import ExecutionConfig, _apply_artwork, execute_consolidation_plan @@ -157,6 +158,81 @@ def test_playlist_rewrite_validation_dry_run(tmp_path): assert (tmp_path / "loser.mp3").exists() +def test_execute_from_preview_output_path(tmp_path): + playlists_dir = tmp_path / "Playlists" + playlists_dir.mkdir() + winner = tmp_path / "winner.flac" + loser = tmp_path / "loser.mp3" + winner.write_text("winner") + loser.write_text("loser") + + playlist = playlists_dir / "mix.m3u" + playlist.write_text(os.path.relpath(str(loser), playlists_dir) + "\n") + + snapshot = {str(winner): _snapshot(winner), str(loser): _snapshot(loser)} + plan = ConsolidationPlan( + groups=[ + GroupPlan( + group_id="g-preview", + winner_path=str(winner), + losers=[str(loser)], + planned_winner_tags={"title": "Winner"}, + winner_current_tags={"title": "Old"}, + current_tags={str(winner): {"title": "Old"}, str(loser): {"title": "Old"}}, + metadata_changes={"title": {"from": "Old", "to": "Winner"}}, + winner_quality={"reasons": ["test"]}, + artwork=[], + artwork_candidates=[], + chosen_artwork_source={}, + artwork_status="unchanged", + loser_disposition={str(loser): "quarantine"}, + playlist_rewrites={str(loser): str(winner)}, + playlist_impact=PlaylistImpact(playlists=1, entries=1), + review_flags=[], + context_summary={"album": [], "single": [], "unknown": [str(winner)]}, + context_evidence={str(winner): [], str(loser): []}, + tag_source=str(winner), + placeholders_present=False, + tag_source_reason="winner", + tag_source_evidence=[], + track_quality={str(winner): {}, str(loser): {}}, + group_confidence="High", + group_match_type="Exact", + grouping_metadata_key=("artist", "title"), + grouping_thresholds={"exact": 0.01, "near": 0.03}, + grouping_decisions=[], + artwork_evidence=[], + fingerprint_distances={}, + library_state=snapshot, + ) + ] + ) + + preview_path = tmp_path / "duplicate_preview.json" + export_consolidation_preview(plan, str(preview_path)) + + cfg = _config( + tmp_path, + allow_review_required=True, + dry_run_execute=True, + ) + + result = execute_consolidation_plan(str(preview_path), cfg) + assert result.success is True + assert result.playlist_reports + assert result.report_paths.get("audit") + + +def test_invalid_preview_output_fails_safely(tmp_path): + preview_path = tmp_path / "duplicate_preview.json" + preview_path.write_text("{\"plan\": \"invalid\"}") + + cfg = _config(tmp_path) + + result = execute_consolidation_plan(str(preview_path), cfg) + assert result.success is False + assert any("Invalid consolidation plan" in action.detail for action in result.actions) + def test_missing_tags_and_placeholders_require_review(tmp_path): winner = tmp_path / "winner.flac" loser = tmp_path / "loser.flac" From 07a359c44e80908d7139b6168cd3da1928cead98 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 28 Dec 2025 16:17:54 -0500 Subject: [PATCH 173/606] Fix temp file handling for artwork and metadata --- duplicate_consolidation_executor.py | 188 ++++++++++++++++------------ 1 file changed, 107 insertions(+), 81 deletions(-) diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index 557f7e7..2d9cbba 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -132,11 +132,10 @@ def _atomic_write_text(path: str, content: str, *, encoding: str = "utf-8") -> N safe_directory = ensure_long_path(directory) safe_path = ensure_long_path(path) os.makedirs(safe_directory, exist_ok=True) - tmp = tempfile.NamedTemporaryFile(delete=False, dir=safe_directory) - tmp_path = tmp.name + with tempfile.NamedTemporaryFile("w", delete=False, dir=safe_directory, encoding=encoding) as tmp: + tmp.write(content) + tmp_path = tmp.name try: - with open(tmp_path, "w", encoding=encoding) as handle: - handle.write(content) os.replace(ensure_long_path(tmp_path), safe_path) finally: if os.path.exists(tmp_path): @@ -249,19 +248,22 @@ def _extract_artwork_bytes(path: str) -> bytes | None: if MutagenFile is None: return None audio = MutagenFile(ensure_long_path(path)) - if audio is None: - return None - pictures = getattr(audio, "pictures", None) - if pictures: - pic = pictures[0] - return bytes(pic.data) if hasattr(pic, "data") else bytes(pic) - tags = getattr(audio, "tags", {}) or {} - for key in list(tags.keys()): - if key.startswith("APIC"): - frame = tags[key] - data = getattr(frame, "data", None) - if data: - return bytes(data) + try: + if audio is None: + return None + pictures = getattr(audio, "pictures", None) + if pictures: + pic = pictures[0] + return bytes(pic.data) if hasattr(pic, "data") else bytes(pic) + tags = getattr(audio, "tags", {}) or {} + for key in list(tags.keys()): + if key.startswith("APIC"): + frame = tags[key] + data = getattr(frame, "data", None) + if data: + return bytes(data) + finally: + _close_mutagen_audio(audio) return None @@ -273,11 +275,10 @@ def _apply_artwork(action: ArtworkDirective) -> tuple[bool, str]: sidecar = f"{action.target}.artwork" directory = os.path.dirname(sidecar) os.makedirs(directory, exist_ok=True) - tmp = tempfile.NamedTemporaryFile(delete=False, dir=directory) - tmp_path = tmp.name + with tempfile.NamedTemporaryFile("wb", delete=False, dir=directory) as tmp: + tmp.write(payload) + tmp_path = tmp.name try: - with open(tmp_path, "wb") as handle: - handle.write(payload) os.replace(tmp_path, sidecar) finally: if os.path.exists(tmp_path): @@ -291,55 +292,58 @@ def _apply_artwork(action: ArtworkDirective) -> tuple[bool, str]: target_path = ensure_long_path(action.target) audio = MutagenFile(target_path) - if audio is None: - return False, "Unsupported audio format for embedding." - - ext = os.path.splitext(action.source)[1].lower() - mime = "image/jpeg" - if ext == ".png": - mime = "image/png" - try: - if hasattr(audio, "pictures"): - # FLAC/APE: replace front cover while preserving other pictures. - from mutagen.flac import Picture # type: ignore - - existing = [p for p in getattr(audio, "pictures") if getattr(p, "type", None) != 3] - pic = Picture() - pic.data = payload - pic.type = 3 - pic.mime = mime - pic.desc = "Front cover" - audio.clear_pictures() - for p in existing: - audio.add_picture(p) - audio.add_picture(pic) - audio.save() - return True, "Embedded FLAC/APE picture." - if audio.__class__.__module__.startswith("mutagen.mp4"): - from mutagen.mp4 import MP4, MP4Cover # type: ignore - - mp4 = audio if isinstance(audio, MP4) else MP4(target_path) - cover_format = MP4Cover.FORMAT_PNG if mime == "image/png" else MP4Cover.FORMAT_JPEG - mp4["covr"] = [MP4Cover(payload, imageformat=cover_format)] - mp4.save() - return True, "Embedded MP4 cover atom." - # Default to ID3/APIC handling - from mutagen.id3 import APIC, ID3, ID3NoHeaderError # type: ignore + if audio is None: + return False, "Unsupported audio format for embedding." + + ext = os.path.splitext(action.source)[1].lower() + mime = "image/jpeg" + if ext == ".png": + mime = "image/png" try: - tags = ID3(target_path) - except ID3NoHeaderError: - tags = ID3() - preserved = [frame for frame in tags.getall("APIC") if getattr(frame, "type", None) != 3] - tags.delall("APIC") - for frame in preserved: - tags.add(frame) - tags.add(APIC(encoding=3, mime=mime, type=3, desc="Front cover", data=payload)) - tags.save(target_path) - return True, "Embedded ID3 cover art." - except Exception as exc: - return False, f"Artwork embed failed: {exc}" + if hasattr(audio, "pictures"): + # FLAC/APE: replace front cover while preserving other pictures. + from mutagen.flac import Picture # type: ignore + + existing = [p for p in getattr(audio, "pictures") if getattr(p, "type", None) != 3] + pic = Picture() + pic.data = payload + pic.type = 3 + pic.mime = mime + pic.desc = "Front cover" + audio.clear_pictures() + for p in existing: + audio.add_picture(p) + audio.add_picture(pic) + audio.save() + return True, "Embedded FLAC/APE picture." + if audio.__class__.__module__.startswith("mutagen.mp4"): + from mutagen.mp4 import MP4, MP4Cover # type: ignore + + mp4 = audio if isinstance(audio, MP4) else MP4(target_path) + cover_format = MP4Cover.FORMAT_PNG if mime == "image/png" else MP4Cover.FORMAT_JPEG + mp4["covr"] = [MP4Cover(payload, imageformat=cover_format)] + mp4.save() + return True, "Embedded MP4 cover atom." + # Default to ID3/APIC handling + from mutagen.id3 import APIC, ID3, ID3NoHeaderError # type: ignore + + try: + tags = ID3(target_path) + except ID3NoHeaderError: + tags = ID3() + preserved = [frame for frame in tags.getall("APIC") if getattr(frame, "type", None) != 3] + tags.delall("APIC") + for frame in preserved: + tags.add(frame) + tags.add(APIC(encoding=3, mime=mime, type=3, desc="Front cover", data=payload)) + tags.save(target_path) + return True, "Embedded ID3 cover art." + except Exception as exc: + return False, f"Artwork embed failed: {exc}" + finally: + _close_mutagen_audio(audio) def _apply_metadata(target: str, planned_tags: Mapping[str, object]) -> bool: @@ -351,23 +355,45 @@ def _apply_metadata(target: str, planned_tags: Mapping[str, object]) -> bool: if MutagenFile is None: return True audio = MutagenFile(ensure_long_path(target), easy=True) - if audio is None: + try: + if audio is None: + return True + changed = False + for key, value in planned_tags.items(): + if value is None: + continue + current = audio.tags.get(key) if audio.tags else None + normalized = [str(value)] if not isinstance(value, list) else [str(v) for v in value] + if current != normalized: + audio[key] = normalized + changed = True + if changed: + try: + audio.save() + except Exception: + return False return True - changed = False - for key, value in planned_tags.items(): - if value is None: - continue - current = audio.tags.get(key) if audio.tags else None - normalized = [str(value)] if not isinstance(value, list) else [str(v) for v in value] - if current != normalized: - audio[key] = normalized - changed = True - if changed: + finally: + _close_mutagen_audio(audio) + + +def _close_mutagen_audio(audio: object | None) -> None: + if audio is None: + return + closer = getattr(audio, "close", None) + if callable(closer): try: - audio.save() + closer() except Exception: - return False - return True + pass + fileobj = getattr(audio, "fileobj", None) + if fileobj is not None: + file_close = getattr(fileobj, "close", None) + if callable(file_close): + try: + file_close() + except Exception: + pass def _quarantine_path(path: str, quarantine_root: str, library_root: str) -> str: From 15613c4547933caaa3c8cda0be3881fbe4369c59 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 28 Dec 2025 16:24:19 -0500 Subject: [PATCH 174/606] Fix execution report path handling --- duplicate_consolidation_executor.py | 13 ++++--------- main_gui.py | 11 ++--------- 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index 2d9cbba..6efb9c4 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -120,6 +120,9 @@ class ExecutionResult: report_paths: Dict[str, str] +EXECUTION_REPORT_FILENAME = "execution_report.html" + + def _timestamped_dir(base: str) -> str: ts = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%d_%H%M%S") path = os.path.join(base, f"consolidation_run_{ts}") @@ -150,12 +153,6 @@ def _atomic_write_json(path: str, payload: Mapping[str, object]) -> None: _atomic_write_text(path, text) -def _ensure_html_extension(path: str) -> str: - if path.lower().endswith(".html"): - return path - return f"{path}.html" - - def _iter_playlists(playlists_dir: str) -> Iterable[str]: if not os.path.isdir(playlists_dir): return [] @@ -926,9 +923,7 @@ def _find_loser_references( audit_path = os.path.join(reports_dir, "audit.json") playlist_report_path = os.path.join(reports_dir, "playlist_report.json") quarantine_index_path = os.path.join(reports_dir, "quarantine_index.json") - html_report_path = _ensure_html_extension( - os.path.join(reports_dir, "execution_report") - ) + html_report_path = os.path.join(reports_dir, EXECUTION_REPORT_FILENAME) plan_signature = plan.plan_signature if plan else None source_snapshot = plan.source_snapshot if plan else {} diff --git a/main_gui.py b/main_gui.py index d9f32d0..3cc626d 100644 --- a/main_gui.py +++ b/main_gui.py @@ -2026,7 +2026,7 @@ def finish(result, error: Exception | None = None) -> None: status = "Executed" if result.success else "Execution failed" self._set_status(status, progress=100) self._log_action(f"Execution complete: {'success' if result.success else 'failed'}") - report_path = self._normalize_html_report_path(result.report_paths.get("html_report")) + report_path = result.report_paths.get("html_report") self._log_action(f"Execution report: {report_path}") if report_path and os.path.exists(ensure_long_path(report_path)): self.execution_report_path = report_path @@ -2037,7 +2037,7 @@ def finish(result, error: Exception | None = None) -> None: if not result.success: report_line = "" if report_path: - if os.path.exists(report_path): + if os.path.exists(ensure_long_path(report_path)): report_line = f"\n\nReport (HTML): {report_path}" else: report_line = ( @@ -2115,13 +2115,6 @@ def _open_preview_output(self) -> None: empty_message="No preview has been generated yet.", ) - def _normalize_html_report_path(self, report_path: str | None) -> str | None: - if not report_path: - return None - if report_path.lower().endswith(".html"): - return report_path - return f"{report_path}.html" - def _open_execution_report(self) -> None: self._open_local_html( self.execution_report_path, From 73db7bcaf38321b176596251a11ba96e420fb903 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 28 Dec 2025 16:44:53 -0500 Subject: [PATCH 175/606] Handle unsupported metadata keys during consolidation --- duplicate_consolidation_executor.py | 35 +++++++++++++++++++---------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index 6efb9c4..b023c13 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -121,6 +121,7 @@ class ExecutionResult: EXECUTION_REPORT_FILENAME = "execution_report.html" +INTERNAL_TAG_KEYS = {"album_type"} def _timestamped_dir(base: str) -> str: @@ -343,33 +344,43 @@ def _apply_artwork(action: ArtworkDirective) -> tuple[bool, str]: _close_mutagen_audio(audio) -def _apply_metadata(target: str, planned_tags: Mapping[str, object]) -> bool: +def _apply_metadata(target: str, planned_tags: Mapping[str, object]) -> tuple[bool, str]: meta_path = f"{target}.metadata.json" try: _atomic_write_json(meta_path, dict(planned_tags)) except Exception: - return False + return False, "Failed to write metadata sidecar." if MutagenFile is None: - return True + return True, "Mutagen unavailable; wrote metadata sidecar only." audio = MutagenFile(ensure_long_path(target), easy=True) try: if audio is None: - return True + return True, "Unsupported audio format; wrote metadata sidecar only." changed = False + skipped: List[str] = [] for key, value in planned_tags.items(): if value is None: continue - current = audio.tags.get(key) if audio.tags else None + if key in INTERNAL_TAG_KEYS: + skipped.append(key) + continue normalized = [str(value)] if not isinstance(value, list) else [str(v) for v in value] - if current != normalized: - audio[key] = normalized - changed = True + try: + current = audio.tags.get(key) if audio.tags else None + if current != normalized: + audio[key] = normalized + changed = True + except Exception: + skipped.append(key) if changed: try: audio.save() except Exception: - return False - return True + return False, "Failed to save metadata tags." + if skipped: + skipped_list = ", ".join(sorted(set(skipped))) + return True, f"Metadata normalized; skipped unsupported keys: {skipped_list}." + return True, "Metadata normalized." finally: _close_mutagen_audio(audio) @@ -823,11 +834,11 @@ def _find_loser_references( elif config.apply_metadata: for target, tags in planned_tags.items(): _check_cancel("metadata") - ok = _apply_metadata(target, tags) + ok, detail = _apply_metadata(target, tags) status = "success" if ok else "failed" if not ok: success = False - _record("metadata", target, status, "Metadata normalized.", group_id=path_to_group.get(target)) + _record("metadata", target, status, detail, group_id=path_to_group.get(target)) if not ok: raise RuntimeError(f"Metadata normalization failed for {target}") From e1af75b0cf152725c927657525893f5208b31a8d Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 28 Dec 2025 17:10:39 -0500 Subject: [PATCH 176/606] Refactor execution report layout --- duplicate_consolidation_executor.py | 215 ++++++++++++++++++++++++---- 1 file changed, 191 insertions(+), 24 deletions(-) diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index b023c13..aba5bc4 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -976,60 +976,227 @@ def _find_loser_references( for gid in group_ids: actions_by_group.setdefault(gid, []).append(action) + def _basename(path: str) -> str: + return os.path.basename(path) or path + + def _group_state(group_actions: List[ExecutionAction]) -> str: + statuses = {act.status for act in group_actions} + for state in ("failed", "blocked", "cancelled"): + if state in statuses: + return state + if statuses == {"skipped"}: + return "skipped" + return "success" if "success" in statuses else "unknown" + + def _action_badges(group_actions: List[ExecutionAction]) -> List[str]: + badges: List[str] = [] + meta_actions = [act for act in group_actions if act.step == "metadata"] + meta_success = sum(1 for act in meta_actions if act.status == "success") + meta_skipped = sum(1 for act in meta_actions if act.status == "skipped") + meta_failed = sum(1 for act in meta_actions if act.status == "failed") + if meta_success: + badges.append(f"metadata normalized ({meta_success})") + if meta_skipped: + badges.append(f"metadata skipped ({meta_skipped})") + if meta_failed: + badges.append(f"metadata failed ({meta_failed})") + + cleanup_actions = [act for act in group_actions if act.step == "loser_cleanup"] + quarantined = sum( + 1 + for act in cleanup_actions + if act.metadata.get("disposition") == "quarantine" + ) + deleted = sum( + 1 + for act in cleanup_actions + if act.metadata.get("disposition") == "delete" + ) + retained = sum( + 1 + for act in cleanup_actions + if act.metadata.get("disposition") == "retain" + ) + if quarantined: + badges.append(f"quarantined ({quarantined})") + if deleted: + badges.append(f"deleted ({deleted})") + if retained: + badges.append(f"retained ({retained})") + + art_actions = [act for act in group_actions if act.step == "artwork"] + art_success = sum(1 for act in art_actions if act.status == "success") + if art_success: + badges.append(f"artwork copied ({art_success})") + return badges + + def _format_metadata_notes(meta: Dict[str, object]) -> str: + notes = [] + for key, value in meta.items(): + if key in {"group_id", "group_ids"}: + continue + if isinstance(value, (list, tuple, set)): + rendered = ", ".join(str(v) for v in value) + else: + rendered = str(value) + notes.append(f"{key}: {rendered}") + return " | ".join(notes) + + metadata_actions = [act for act in actions if act.step == "metadata"] + metadata_success = sum(1 for act in metadata_actions if act.status == "success") + metadata_failed = sum(1 for act in metadata_actions if act.status == "failed") + metadata_skipped = sum(1 for act in metadata_actions if act.status == "skipped") + quarantined_count = sum( + 1 for dest in quarantine_index.values() if dest not in {"retained", "deleted"} + ) + deleted_count = sum(1 for dest in quarantine_index.values() if dest == "deleted") + groups_processed = len(group_lookup) + winners_kept = groups_processed + html_lines = [ "Duplicate Consolidation Execution", "", "

    Execution Report

    ", - f"

    Status: {'success' if success else 'failed'}

    ", - f"

    Plan signature: {plan_signature or computed_signature}

    ", - f"

    Reports directory: {reports_dir}

    ", + "
    ", + "
    ", + f"
    Overall status
    " + f"
    {'success' if success else 'failed'}
    ", + f"
    Groups processed
    " + f"
    {groups_processed}
    ", + f"
    Winners kept
    " + f"
    {winners_kept}
    ", + f"
    Files quarantined
    " + f"
    {quarantined_count}" + f"{' (deleted: ' + str(deleted_count) + ')' if deleted_count else ''}
    ", + "
    Metadata operations
    " + f"
    {metadata_success} ok / {metadata_failed} failed / {metadata_skipped} skipped
    ", + "
    ", + "
    ", + f"Plan signature: {plan_signature or computed_signature}
    ", + f"Reports directory: {reports_dir}", + "
    ", + "
    ", + "

    Duplicate Groups

    ", ] for gid in sorted([g for g in actions_by_group.keys() if g != "__general__"]): grp = group_lookup.get(gid) winner_path = getattr(grp, "winner_path", "Unknown winner") - html_lines.append(f"

    Group {gid}

    ") - html_lines.append(f"

    Winner: {winner_path}

    ") - html_lines.append("
      ") - for act in actions_by_group.get(gid, []): + group_actions = actions_by_group.get(gid, []) + state = _group_state(group_actions) + badges = _action_badges(group_actions) + badge_html = " ".join([f"{badge}" for badge in badges]) + html_lines.append("
      ") + html_lines.append( + "" + "
      " + f"Group {gid}" + f"• Winner: {_basename(winner_path)}" + f"" + f"{state}" + f"{badge_html}" + "
      " + "
      " + ) + html_lines.append("
      ") + html_lines.append(f"
      Winner path: {winner_path}
      ") + html_lines.append("") + html_lines.append( + "" + ) + for act in group_actions: + meta_notes = _format_metadata_notes(act.metadata) + notes = act.detail + if meta_notes: + notes = f"{notes}
      {meta_notes}
      " html_lines.append( - f"
    • {act.step} — {act.target}: " - f"{act.status} ({act.detail})
    • " + "" + f"" + f"" + f"" + f"" + "" ) - html_lines.append("") + html_lines.append("
      OperationTargetStatusNotes
      {act.step}{_basename(act.target)}{act.status}{notes}
      ") if actions_by_group.get("__general__"): - html_lines.append("

      General Actions

        ") + html_lines.append("
        ") + html_lines.append("
        General Actions
        ") + html_lines.append("
        ") + html_lines.append( + "" + ) for act in actions_by_group["__general__"]: + meta_notes = _format_metadata_notes(act.metadata) + notes = act.detail + if meta_notes: + notes = f"{notes}
        {meta_notes}
        " html_lines.append( - f"
      • {act.step} — {act.target}: " - f"{act.status} ({act.detail})
      • " + "" + f"" + f"" + f"" + f"" + "" ) - html_lines.append("") + html_lines.append("
        OperationTargetStatusNotes
        {act.step}{_basename(act.target)}{act.status}{notes}
        ") if playlist_results: - html_lines.append("

        Playlist Changes

          ") + html_lines.append("
          ") + html_lines.append("
          Playlist Changes
          ") + html_lines.append("
          ") + html_lines.append( + "" + ) for res in playlist_results: label = res.playlist if res.original_playlist: label = f"{res.original_playlist} (validated copy: {res.playlist})" html_lines.append( - f"
        • {label} → {res.status} " - f"(replaced {res.replaced_entries}); backup: {res.backup_path}
        • " + "" + f"" + f"" + f"" + "" ) - html_lines.append("") + html_lines.append("
          PlaylistStatusDetails
          {label}{res.status}replaced {res.replaced_entries}; backup: {res.backup_path}
          ") + if quarantine_index: - html_lines.append("

          Quarantined/Deleted

            ") + html_lines.append("
            ") + html_lines.append("
            Quarantined Files
            ") + html_lines.append("
            ") + html_lines.append("") for loser, dest in quarantine_index.items(): - html_lines.append(f"
          • {loser} → {dest}
          • ") - html_lines.append("") + html_lines.append( + f"" + ) + html_lines.append("
            OriginalDestination
            {loser}{dest}
            ") html_lines.append("") _atomic_write_text(html_report_path, "\n".join(html_lines)) if not os.path.exists(ensure_long_path(html_report_path)): From bd02151d4dab27a2aa45f19c5cd557dcc8d78a44 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 28 Dec 2025 17:17:45 -0500 Subject: [PATCH 177/606] Consolidate duplicate finder execution reports --- duplicate_consolidation_executor.py | 34 +++++++++++------------------ 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index aba5bc4..e2e8146 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -345,17 +345,12 @@ def _apply_artwork(action: ArtworkDirective) -> tuple[bool, str]: def _apply_metadata(target: str, planned_tags: Mapping[str, object]) -> tuple[bool, str]: - meta_path = f"{target}.metadata.json" - try: - _atomic_write_json(meta_path, dict(planned_tags)) - except Exception: - return False, "Failed to write metadata sidecar." if MutagenFile is None: - return True, "Mutagen unavailable; wrote metadata sidecar only." + return True, "Mutagen unavailable; metadata not written." audio = MutagenFile(ensure_long_path(target), easy=True) try: if audio is None: - return True, "Unsupported audio format; wrote metadata sidecar only." + return True, "Unsupported audio format; metadata not written." changed = False skipped: List[str] = [] for key, value in planned_tags.items(): @@ -931,31 +926,30 @@ def _find_loser_references( success = False _record("execution", "execution", "failed", f"Execution failed: {exc}") finally: - audit_path = os.path.join(reports_dir, "audit.json") - playlist_report_path = os.path.join(reports_dir, "playlist_report.json") - quarantine_index_path = os.path.join(reports_dir, "quarantine_index.json") + consolidated_report_path = os.path.join(reports_dir, "execution_report.json") html_report_path = os.path.join(reports_dir, EXECUTION_REPORT_FILENAME) plan_signature = plan.plan_signature if plan else None source_snapshot = plan.source_snapshot if plan else {} - audit_payload = { + consolidated_payload = { "success": success, "actions": [a.to_dict() for a in actions], "generated_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), "plan_signature": plan_signature or computed_signature, "computed_signature": computed_signature, "source_snapshot": {k: dict(v) for k, v in source_snapshot.items()}, - } - playlist_payload = { - "backups_dir": backups_dir, + "plan": plan.to_dict() if plan else None, "playlists": [p.to_dict() for p in playlist_results], + "quarantine_index": dict(quarantine_index), + "metadata_plans": {path: dict(tags) for path, tags in planned_tags.items()}, + "run_paths": { + "reports_dir": reports_dir, + "backups_dir": backups_dir, + }, } - quarantine_payload = dict(quarantine_index) try: - _atomic_write_json(audit_path, audit_payload) - _atomic_write_json(playlist_report_path, playlist_payload) - _atomic_write_json(quarantine_index_path, quarantine_payload) + _atomic_write_json(consolidated_report_path, consolidated_payload) except Exception: pass @@ -1205,9 +1199,7 @@ def _format_metadata_notes(meta: Dict[str, object]) -> str: log(f"Report write failed: unable to create HTML report at {html_report_path} ({exc})") report_paths = { - "audit": audit_path, - "playlist_report": playlist_report_path, - "quarantine_index": quarantine_index_path, + "json_report": consolidated_report_path, "html_report": html_report_path, "reports_root": reports_dir, "backups_root": backups_dir, From a5f5572de72a20c9e29bc20b5ce23400676400cf Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 28 Dec 2025 17:25:06 -0500 Subject: [PATCH 178/606] Update execution report template mapping --- duplicate_consolidation_executor.py | 715 ++++++++++++++++++++++++---- 1 file changed, 610 insertions(+), 105 deletions(-) diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index e2e8146..6dcf735 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -14,6 +14,7 @@ import datetime import hashlib +import html import importlib.util import json import os @@ -1046,57 +1047,383 @@ def _format_metadata_notes(meta: Dict[str, object]) -> str: deleted_count = sum(1 for dest in quarantine_index.values() if dest == "deleted") groups_processed = len(group_lookup) winners_kept = groups_processed + generated_at_iso = datetime.datetime.now(datetime.timezone.utc).isoformat() + host_name = os.uname().nodename + + def _status_badge_class(status: str) -> str: + if status == "success": + return "ok" + if status == "failed": + return "fail" + return "warn" + + def _format_notes(detail: str, meta_notes: str) -> str: + detail_html = html.escape(detail) + if meta_notes: + meta_html = html.escape(meta_notes) + return f"{detail_html}
            {meta_html}
            " + return detail_html + + def _group_type_hint(group_actions: List[ExecutionAction]) -> str: + if not group_actions: + return "unknown" + if all(act.step == "metadata" for act in group_actions): + return "metadata-only" + return "mixed" + + def _group_quarantine_count(group_actions: List[ExecutionAction]) -> int: + return sum( + 1 + for act in group_actions + if act.step == "loser_cleanup" + and act.metadata.get("disposition") == "quarantine" + ) + + def _group_has_quarantine(group_actions: List[ExecutionAction]) -> bool: + return _group_quarantine_count(group_actions) > 0 + + def _group_has_failure(group_actions: List[ExecutionAction]) -> bool: + return any( + act.status in {"failed", "blocked", "cancelled"} for act in group_actions + ) + + def _run_status() -> tuple[str, str, str]: + status = "success" if success else "failed" + status_class = "ok" if success else "fail" + emoji = "✅" if success else "❌" + return status, status_class, f"{emoji} {status}" html_lines = [ - "Duplicate Consolidation Execution", + "", + "", + "", + "", + "", + "Execution Report", "", + ":root{", + "--bg:#ffffff;", + "--text:#111;", + "--muted:#666;", + "--border:#e6e6e6;", + "--card:#fafafa;", + "--good:#1b7a1b;", + "--bad:#a30000;", + "--warn:#8a5b00;", + "--chip:#f2f2f2;", + "--mono: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, \"Liberation Mono\", \"Courier New\", monospace;", + "--sans: ui-sans-serif, system-ui, -apple-system, Segoe UI, Roboto, Helvetica, Arial, \"Apple Color Emoji\",\"Segoe UI Emoji\";", + "}", + "* { box-sizing: border-box; }", + "body{", + "margin: 18px;", + "font-family: var(--sans);", + "background: var(--bg);", + "color: var(--text);", + "line-height: 1.35;", + "}", + "header{", + "display:flex;", + "flex-wrap:wrap;", + "gap:12px 16px;", + "align-items:baseline;", + "justify-content:space-between;", + "margin-bottom: 12px;", + "}", + "h1{", + "font-size: 20px;", + "margin: 0;", + "letter-spacing: .2px;", + "}", + ".row{", + "display:flex;", + "flex-wrap:wrap;", + "gap:10px;", + "align-items:center;", + "}", + ".card{", + "border:1px solid var(--border);", + "border-radius: 10px;", + "padding: 12px;", + "background: var(--card);", + "}", + ".summary{", + "display:grid;", + "grid-template-columns: 1fr;", + "gap: 10px;", + "margin-bottom: 14px;", + "}", + "@media (min-width: 900px){", + ".summary{ grid-template-columns: 2fr 1fr; }", + "}", + ".kv{", + "display:grid;", + "grid-template-columns: 160px 1fr;", + "gap:6px 10px;", + "align-items:start;", + "}", + ".k{ color: var(--muted); font-size: 12px; }", + ".v{ font-size: 13px; }", + ".mono{ font-family: var(--mono); font-size: 12px; }", + ".path{", + "font-family: var(--mono);", + "font-size: 12px;", + "word-break: break-all;", + "}", + ".status{", + "font-weight: 600;", + "display:inline-flex;", + "align-items:center;", + "gap:6px;", + "}", + ".ok{ color: var(--good); }", + ".fail{ color: var(--bad); }", + ".warn{ color: var(--warn); }", + ".pill{", + "display:inline-flex;", + "align-items:center;", + "gap:8px;", + "padding: 6px 10px;", + "border-radius: 999px;", + "background: var(--chip);", + "border: 1px solid var(--border);", + "font-size: 12px;", + "white-space: nowrap;", + "}", + ".pill strong{ font-weight: 700; }", + ".controls{", + "display:flex;", + "flex-wrap:wrap;", + "gap:10px;", + "align-items:center;", + "justify-content:space-between;", + "margin: 14px 0 10px;", + "}", + ".controls .left, .controls .right{", + "display:flex; flex-wrap:wrap; gap:10px; align-items:center;", + "}", + "input[type='search']{", + "padding: 9px 10px;", + "border:1px solid var(--border);", + "border-radius: 10px;", + "width: min(520px, 92vw);", + "font-size: 13px;", + "background: #fff;", + "}", + "select{", + "padding: 9px 10px;", + "border:1px solid var(--border);", + "border-radius: 10px;", + "font-size: 13px;", + "background: #fff;", + "}", + "button.copy{", + "border:1px solid var(--border);", + "background:#fff;", + "border-radius: 10px;", + "padding: 8px 10px;", + "font-size: 12px;", + "cursor:pointer;", + "}", + "button.copy:active{ transform: translateY(1px); }", + "details.group{", + "border:1px solid var(--border);", + "border-radius: 10px;", + "background: #fff;", + "margin-bottom: 10px;", + "overflow:hidden;", + "}", + "details.group[open]{ box-shadow: 0 1px 0 rgba(0,0,0,.03); }", + "summary.group-summary{", + "list-style:none;", + "cursor:pointer;", + "padding: 12px 12px;", + "display:grid;", + "grid-template-columns: 1fr;", + "gap:8px;", + "background: #fff;", + "}", + "summary.group-summary::-webkit-details-marker{ display:none; }", + ".group-top{", + "display:flex;", + "flex-wrap:wrap;", + "gap:10px 12px;", + "align-items:center;", + "justify-content:space-between;", + "}", + ".group-title{", + "display:flex;", + "align-items:baseline;", + "gap:10px;", + "min-width: 240px;", + "}", + ".gid{", + "font-family: var(--mono);", + "font-size: 12px;", + "color: var(--muted);", + "}", + ".winner-short{", + "font-weight: 650;", + "font-size: 13px;", + "max-width: 74ch;", + "overflow:hidden;", + "text-overflow: ellipsis;", + "white-space: nowrap;", + "}", + ".chips{", + "display:flex;", + "flex-wrap:wrap;", + "gap:8px;", + "justify-content:flex-end;", + "}", + ".group-body{", + "border-top: 1px solid var(--border);", + "background: var(--card);", + "padding: 12px;", + "display:grid;", + "grid-template-columns: 1fr;", + "gap: 10px;", + "}", + "@media (min-width: 980px){", + ".group-body{ grid-template-columns: 1fr 1.2fr; }", + "}", + ".section h3{", + "font-size: 12px;", + "margin: 0 0 8px;", + "color: var(--muted);", + "text-transform: uppercase;", + "letter-spacing: .08em;", + "}", + "table.ops{", + "width:100%;", + "border-collapse: collapse;", + "background:#fff;", + "border:1px solid var(--border);", + "border-radius: 10px;", + "overflow:hidden;", + "}", + "table.ops th, table.ops td{", + "border-bottom:1px solid var(--border);", + "padding: 9px 10px;", + "font-size: 12px;", + "vertical-align: top;", + "}", + "table.ops th{", + "text-align:left;", + "color: var(--muted);", + "background: #fcfcfc;", + "font-weight: 650;", + "}", + "table.ops tr:last-child td{ border-bottom: none; }", + ".op-key{", + "font-family: var(--mono);", + "font-size: 12px;", + "color:#222;", + "white-space:nowrap;", + "}", + ".note{", + "color: var(--muted);", + "}", + ".badge{", + "display:inline-flex;", + "align-items:center;", + "padding: 3px 8px;", + "border-radius: 999px;", + "border:1px solid var(--border);", + "font-size: 12px;", + "font-weight: 650;", + "background: #fff;", + "white-space: nowrap;", + "}", + ".badge.ok{ border-color: rgba(27,122,27,.25); }", + ".badge.fail{ border-color: rgba(163,0,0,.25); }", + "details.quarantine{", + "border:1px solid var(--border);", + "border-radius: 10px;", + "background:#fff;", + "margin-top: 14px;", + "overflow:hidden;", + "}", + "summary.quarantine-summary{", + "cursor:pointer;", + "padding: 12px;", + "list-style:none;", + "}", + "summary.quarantine-summary::-webkit-details-marker{ display:none; }", + "ul.q-list{", + "margin:0;", + "padding: 0 12px 12px 28px;", + "color:#222;", + "}", + "ul.q-list li{", + "font-family: var(--mono);", + "font-size: 12px;", + "word-break: break-all;", + "margin: 6px 0;", + "}", + ".muted{ color: var(--muted); }", + ".tiny{ font-size: 12px; }", + ".hidden{ display:none !important; }", + "", + "", + "", + "
            ", "

            Execution Report

            ", - "
            ", - "
            ", - f"
            Overall status
            " - f"
            {'success' if success else 'failed'}
            ", - f"
            Groups processed
            " - f"
            {groups_processed}
            ", - f"
            Winners kept
            " - f"
            {winners_kept}
            ", - f"
            Files quarantined
            " - f"
            {quarantined_count}" - f"{' (deleted: ' + str(deleted_count) + ')' if deleted_count else ''}
            ", - "
            Metadata operations
            " - f"
            {metadata_success} ok / {metadata_failed} failed / {metadata_skipped} skipped
            ", + f"
            " + f"{_run_status()[2]}
            ", + "
            ", + "
            ", + "
            ", + "
            ", + "
            ", + "Groups 0", + "Winners 0", + "Quarantined 0", + "Ops 0/0 ok", + "
            ", + "
            ", + "", + "", + "
            ", + "
            ", + "
            ", + "
            ", + "
            Plan signature
            ", + f"
            {html.escape(plan_signature or computed_signature)}
            ", + "
            Reports directory
            ", + f"
            {html.escape(reports_dir)}
            ", + "
            ", + "
            ", + "
            ", + "
            ", + "
            Generated
            ", + f"
            {html.escape(generated_at_iso)}
            ", + "
            Host
            ", + f"
            {html.escape(host_name)}
            ", + "
            Notes
            ", + "
            ", + "Groups are collapsed by default. Full paths and verbose messages are in “Context”.", "
            ", - "
            ", - f"Plan signature: {plan_signature or computed_signature}
            ", - f"Reports directory: {reports_dir}", "
            ", "
            ", - "

            Duplicate Groups

            ", + "
            ", + "
            ", + "
            ", + "", + "", + "
            ", + "
            ", + "0 visible", + "", + "", + "
            ", + "
            ", + "
            ", ] for gid in sorted([g for g in actions_by_group.keys() if g != "__general__"]): @@ -1105,92 +1432,270 @@ def _format_metadata_notes(meta: Dict[str, object]) -> str: group_actions = actions_by_group.get(gid, []) state = _group_state(group_actions) badges = _action_badges(group_actions) - badge_html = " ".join([f"{badge}" for badge in badges]) - html_lines.append("
            ") + group_quarantine_count = _group_quarantine_count(group_actions) + group_type_hint = _group_type_hint(group_actions) + group_has_quarantine = _group_has_quarantine(group_actions) + group_has_failure = _group_has_failure(group_actions) + search_tokens = [gid, winner_path, _basename(winner_path), state] + search_tokens.extend(badges) + for act in group_actions: + search_tokens.extend([act.step, act.target, act.status, act.detail]) + meta_notes = _format_metadata_notes(act.metadata) + if meta_notes: + search_tokens.append(meta_notes) + search_text = " ".join(str(token) for token in search_tokens if token) + summary_line = ( + f"Status: {state}. " + "; ".join(badges) if badges else f"Status: {state}." + ) html_lines.append( - "" - "
            " - f"Group {gid}" - f"• Winner: {_basename(winner_path)}" - f"" - f"{state}" - f"{badge_html}" - "
            " - "
            " + "
            " + ) + html_lines.append("") + html_lines.append("
            ") + html_lines.append("
            ") + html_lines.append(f"Group {html.escape(gid)}") + html_lines.append( + "{html.escape(_basename(winner_path))}" + ) + html_lines.append("
            ") + html_lines.append("
            ") + html_lines.append( + f"" + f"{html.escape(state)}" ) - html_lines.append("
            ") - html_lines.append(f"
            Winner path: {winner_path}
            ") - html_lines.append("") + for badge in badges: + html_lines.append(f"{html.escape(badge)}") + html_lines.append("") + html_lines.append("") + html_lines.append(f"
            {html.escape(summary_line)}
            ") + html_lines.append("") + html_lines.append("
            ") + html_lines.append("
            ") + html_lines.append("

            Context

            ") + html_lines.append("
            ") + html_lines.append("
            ") + html_lines.append("
            Kept file (winner)
            ") + html_lines.append(f"
            {html.escape(winner_path)}
            ") + html_lines.append("
            Group ID
            ") + html_lines.append(f"
            {html.escape(gid)}
            ") + html_lines.append("
            Debug
            ") html_lines.append( - "
            " + "
            Shown for traceability; safe to ignore for normal review.
            " ) + html_lines.append("") + html_lines.append("") + html_lines.append("") + html_lines.append("
            ") + html_lines.append("

            Operations

            ") + html_lines.append("
            OperationTargetStatusNotes
            ") + html_lines.append("") + html_lines.append("") + html_lines.append("") + html_lines.append("") + html_lines.append("") + html_lines.append("") + html_lines.append("") + html_lines.append("") + html_lines.append("") for act in group_actions: meta_notes = _format_metadata_notes(act.metadata) - notes = act.detail - if meta_notes: - notes = f"{notes}
            {meta_notes}
            " + notes = _format_notes(act.detail, meta_notes) + status_class = _status_badge_class(act.status) html_lines.append( "" - f"" - f"" - f"" - f"" + f"" + f"" + f"" + f"" "" ) - html_lines.append("
            OperationTargetStatusNotes
            {act.step}{_basename(act.target)}{act.status}{notes}{html.escape(act.step)}" + f"{html.escape(_basename(act.target))}{html.escape(act.status)}{notes}
            ") - - if actions_by_group.get("__general__"): - html_lines.append("
            ") - html_lines.append("
            General Actions
            ") - html_lines.append("
            ") + html_lines.append("") + html_lines.append("
            ") + html_lines.append("
            ") + html_lines.append("
          ") + html_lines.append("") + + html_lines.append("") + html_lines.append("
          ") + html_lines.append("") + html_lines.append("
          ") + html_lines.append( + "
          Quarantined Files (0)
          " + ) + html_lines.append("
          Collapsed by default
          ") + html_lines.append("
          ") + html_lines.append("
          ") + html_lines.append("
            ") + for loser, dest in quarantine_index.items(): html_lines.append( - "OperationTargetStatusNotes" + f"
          • {html.escape(loser)} → {html.escape(dest)}
          • " + ) + html_lines.append("
          ") + html_lines.append("
          ") + + missing_sections: List[str] = [] + missing_sections.append( + f"Metadata operations: {metadata_success} ok / {metadata_failed} failed / {metadata_skipped} skipped" + ) + if deleted_count: + missing_sections.append( + f"Files quarantined: {quarantined_count} (deleted: {deleted_count})" ) - for act in actions_by_group["__general__"]: + else: + missing_sections.append(f"Files quarantined: {quarantined_count}") + + general_actions = actions_by_group.get("__general__", []) + playlist_results = playlist_results or [] + if general_actions or playlist_results: + missing_sections.append("") + if general_actions: + missing_sections.append("General Actions") + missing_sections.append("") + missing_sections.append( + "" + ) + missing_sections.append("") + for act in general_actions: meta_notes = _format_metadata_notes(act.metadata) - notes = act.detail - if meta_notes: - notes = f"{notes}
          {meta_notes}
          " - html_lines.append( + notes = _format_notes(act.detail, meta_notes) + status_class = _status_badge_class(act.status) + missing_sections.append( "" - f"" - f"" - f"" - f"" + f"" + f"" + f"" + f"" "" ) - html_lines.append("
          OperationTargetStatusNotes
          {act.step}{_basename(act.target)}{act.status}{notes}{html.escape(act.step)}" + f"{html.escape(_basename(act.target))}{html.escape(act.status)}{notes}
        ") - + missing_sections.append("") if playlist_results: - html_lines.append("
        ") - html_lines.append("
        Playlist Changes
        ") - html_lines.append("
        ") - html_lines.append( - "" + missing_sections.append("Playlist Changes") + missing_sections.append("
        PlaylistStatusDetails
        ") + missing_sections.append( + "" ) + missing_sections.append("") for res in playlist_results: label = res.playlist if res.original_playlist: label = f"{res.original_playlist} (validated copy: {res.playlist})" - html_lines.append( + status_class = _status_badge_class(res.status) + missing_sections.append( "" - f"" - f"" - f"" + f"" + f"" + f"" "" ) - html_lines.append("
        PlaylistStatusDetails
        {label}{res.status}replaced {res.replaced_entries}; backup: {res.backup_path}{html.escape(label)}{html.escape(res.status)}replaced {res.replaced_entries}; backup: {html.escape(res.backup_path)}
        ") - - if quarantine_index: - html_lines.append("
        ") - html_lines.append("
        Quarantined Files
        ") - html_lines.append("
        ") - html_lines.append("") - for loser, dest in quarantine_index.items(): - html_lines.append( - f"" - ) - html_lines.append("
        OriginalDestination
        {loser}{dest}
        ") + missing_sections.append("") + + if missing_sections: + html_lines.append( + "
        " + "NEED TO INCORPORATE AT LATER DATE" + "
        " + ) + for section in missing_sections: + if not section: + html_lines.append("
        ") + continue + html_lines.append(f"
        {section}
        ") + html_lines.append( + "" + ) html_lines.append("") _atomic_write_text(html_report_path, "\n".join(html_lines)) if not os.path.exists(ensure_long_path(html_report_path)): From 965fee876d80ffd02551cc714a8ce4dbd12fa638 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 28 Dec 2025 17:51:26 -0500 Subject: [PATCH 179/606] Ensure unique duplicate execution report directories --- duplicate_consolidation_executor.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index 6dcf735..71e344b 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -126,10 +126,17 @@ class ExecutionResult: def _timestamped_dir(base: str) -> str: - ts = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%d_%H%M%S") - path = os.path.join(base, f"consolidation_run_{ts}") - os.makedirs(path, exist_ok=True) - return path + os.makedirs(base, exist_ok=True) + ts = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%d_%H%M%S_%f") + for attempt in range(1000): + suffix = f"_{attempt}" if attempt else "" + path = os.path.join(base, f"consolidation_run_{ts}{suffix}") + try: + os.makedirs(path) + except FileExistsError: + continue + return path + raise RuntimeError("Unable to create unique consolidation report directory.") def _atomic_write_text(path: str, content: str, *, encoding: str = "utf-8") -> None: From 24943d26461a13aa66c303fcc408cbc272a89908 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 28 Dec 2025 18:01:46 -0500 Subject: [PATCH 180/606] Update duplicate preview HTML layout --- duplicate_consolidation.py | 127 +++++++++++++++++++++++++++---------- 1 file changed, 94 insertions(+), 33 deletions(-) diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index 2dc523b..0c85c7c 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -1923,78 +1923,137 @@ def _render_artwork_section(group: GroupPlan) -> str: return "".join(lines) review_required = plan.review_required_groups + + def _basename(path: str) -> str: + base = os.path.basename(path) + return base or path + lines = [ "", + "
        ", + "
        ", "

        Duplicate Consolidation Preview

        ", - f"

        Generated at {plan.generated_at.isoformat()}

        ", - f"

        Total groups: {len(plan.groups)} | Review required: {len(review_required)} | " - f"Global flags: {len(plan.review_flags)}

        ", + f"
        Generated at {plan.generated_at.isoformat()}
        ", + "
        ", + f"
        Total groups {len(plan.groups)}
        ", + "
        ", + "
        ", + f"
        Review required
        {len(review_required)}
        ", + f"
        Global flags
        {len(plan.review_flags)}
        ", + "
        ", ] if plan.review_flags: - lines.append("
        Plan Warnings
          ") + lines.append("
          Plan Warnings
            ") for flag in plan.review_flags: lines.append(f"
          • {_html_escape(flag)}
          • ") - lines.append("
        ") + lines.append("
      ") for group in plan.groups: review_badge = ( - "Review required" if group.review_flags else "Ready" + "Review required" + if group.review_flags + else "Ready" ) - lines.append("
      ".format(cls="review" if group.review_flags else "")) + winner_basename = _basename(group.winner_path) + lines.append("
      ") + lines.append("") + lines.append("
      ") + lines.append("
      ") + lines.append(f"Group {group.group_id}") lines.append( - f"
      Group {group.group_id}" - f"
      Confidence: {_html_escape(group.group_confidence)}
      {review_badge}
      " + f"" + f"{_html_escape(winner_basename)}" ) + lines.append("
      ") + lines.append(review_badge) + lines.append("
      ") + lines.append(f"
      Confidence: {_html_escape(group.group_confidence)}
      ") + lines.append("
      ") + + lines.append("
      ") + lines.append("
      ") codec_warning = _render_codec_mix_warning(group) if codec_warning: lines.append(codec_warning) - lines.append(f"

      Winner: {_html_escape(group.winner_path)}

      ") + lines.append("

      Winner

      ") + lines.append(f"
      {_html_escape(group.winner_path)}
      ") lines.append(_render_quality(group)) + lines.append("
      ") lines.append(_render_grouping_evidence(group)) - - lines.append("

      Losers

      ") + lines.append("

      Losers

      ") lines.append(_render_losers(group)) lines.append("
      ") + lines.append("
      ") - lines.append("

      Metadata normalization

      ") + lines.append("
      ") + lines.append("

      Metadata normalization

      ") lines.append(_render_metadata(group.metadata_changes)) - lines.append(f"

      Tag source: {_html_escape(group.tag_source or 'None')} — {_html_escape(group.tag_source_reason)}

      ") + lines.append( + "
      " + f"Tag source: {_html_escape(group.tag_source or 'None')} — " + f"{_html_escape(group.tag_source_reason)}
      " + ) if group.tag_source_evidence: lines.append("
        " + "".join(f"
      • {_html_escape(ev)}
      • " for ev in group.tag_source_evidence) + "
      ") lines.append("
      ") - lines.append("

      Artwork

      ") + lines.append("

      Artwork

      ") lines.append(_render_artwork_section(group)) lines.append("
      ") lines.append( - f"

      Playlist impact: {group.playlist_impact.playlists} playlists, " - f"{group.playlist_impact.entries} entries

      " + "

      Playlist impact

      " + f"
      {group.playlist_impact.playlists} playlists, " + f"{group.playlist_impact.entries} entries
      " ) if group.review_flags: - lines.append("

      Review flags:

        ") + lines.append("

        Review flags

          ") for flag in group.review_flags: lines.append(f"
        • {_html_escape(flag)}
        • ") - lines.append("
        ") + lines.append("
      ") lines.append( - "
      Context
        " + "
        Context & evidence
          " + f"
        • Album tracks: {len(group.context_summary.get('album', []))}
        • " + f"
        • Singles: {len(group.context_summary.get('single', []))}
        • " + f"
        • Unknown: {len(group.context_summary.get('unknown', []))}
        • " @@ -2016,6 +2075,8 @@ def _render_artwork_section(group: GroupPlan) -> str: + "
        " ) lines.append("
      ") + lines.append("
      ") + lines.append("
      ") lines.append("") From a5fd1e887a3fc4f5406ac81cea305515def24b5f Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 28 Dec 2025 18:11:24 -0500 Subject: [PATCH 181/606] Remove deprecated duplicate preview HTML renderer --- duplicate_consolidation.py | 337 ------------------ main_gui.py | 25 +- tests/test_duplicate_consolidation_preview.py | 9 +- 3 files changed, 2 insertions(+), 369 deletions(-) diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index 0c85c7c..01997b2 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -10,7 +10,6 @@ import datetime import hashlib -import html import importlib.util import io import json @@ -1749,342 +1748,6 @@ def build_consolidation_plan( return ConsolidationPlan(groups=plans, review_flags=review_flags, placeholders_present=plan_placeholders) -def _html_escape(text: object) -> str: - return html.escape("" if text is None else str(text)) - - -def render_consolidation_preview(plan: ConsolidationPlan, output_html_path: str) -> str: - """Render an HTML preview summarizing the consolidation plan.""" - - def _render_metadata(changes: Dict[str, Dict[str, object]]) -> str: - if not changes: - return "No tag changes planned." - rows = [] - for key in sorted(changes.keys()): - diff = changes[key] - rows.append( - f"{_html_escape(key)}{_html_escape(diff.get('from') or '—')}" - f"{_html_escape(diff.get('to') or '—')}" - ) - return ( - "" - "" - + "".join(rows) - + "" - ) - - def _render_quality(group: GroupPlan) -> str: - winner = group.winner_path - q = group.track_quality.get(winner, {}) - rows = [ - ("Container / Codec", f"{_html_escape(q.get('container') or '—')} / {_html_escape(q.get('codec') or '—')}"), - ("Bitrate", f"{_html_escape(q.get('bitrate') or '0')} kbps"), - ("Sample Rate", f"{_html_escape(q.get('sample_rate') or '0')} Hz"), - ("Bit Depth", f"{_html_escape(q.get('bit_depth') or '0')} bit"), - ("Channels", _html_escape(q.get("channels") or "—")), - ] - table = ( - "" - "" - + "".join(f"" for label, value in rows) - + "" - ) - reasons = group.winner_quality.get("reasons") or [] - return ( - "
      " - f"

      Winner audio

      " - f"

      Context: {_html_escape(group.winner_quality.get('context', 'unknown'))}

      " - f"{table}" - "

      Why this is the winner

      " - "
        " - + "".join(f"
      • {_html_escape(r)}
      • " for r in reasons) - + "
      " - "
      " - ) - - def _render_codec_mix_warning(group: GroupPlan) -> str: - labels: List[str] = [] - for path, quality in group.track_quality.items(): - container = str(quality.get("container") or "").strip() - if not container: - ext = os.path.splitext(path)[1].replace(".", "").upper() - container = ext or "UNKNOWN" - labels.append(container.upper()) - unique = sorted(set(labels)) - if len(unique) <= 1: - return "" - return f"

      Codec mix warning: {' + '.join(unique)} in group

      " - - def _render_grouping_evidence(group: GroupPlan) -> str: - meta_artist, meta_title = group.grouping_metadata_key - lines = [ - "

      Grouping rationale

      ", - f"

      Metadata gate: artist '{_html_escape(meta_artist)}', title '{_html_escape(meta_title)}'

      ", - f"

      Fingerprint thresholds: exact ≤ {_html_escape(group.grouping_thresholds.get('exact'))}, " - f"near ≤ {_html_escape(group.grouping_thresholds.get('near'))}

      ", - f"

      Grouping type: {_html_escape(group.group_match_type)}

      ", - ] - if group.grouping_decisions: - rows = [] - for decision in group.grouping_decisions: - shared = ", ".join(decision.shared_coarse_keys) if decision.shared_coarse_keys else "—" - rows.append( - "" - f"{_html_escape(decision.candidate_path)}" - f"{_html_escape(decision.anchor_path)}" - f"{_html_escape(decision.coarse_gate)}" - f"{_html_escape(shared)}" - f"{decision.distance_to_anchor:.4f}" - f"{decision.max_group_distance:.4f}" - f"{_html_escape(decision.match_type)}" - "" - ) - lines.append( - "" - "" - "" - + "".join(rows) - + "" - ) - else: - lines.append("No grouping decisions recorded.") - if group.fingerprint_distances: - rows = [] - for left, right_map in sorted(group.fingerprint_distances.items()): - for right, dist in sorted(right_map.items()): - if left >= right: - continue - match_type = "exact" if dist <= group.grouping_thresholds.get("exact", 0.0) else "near" - rows.append( - "" - f"{_html_escape(left)}" - f"{_html_escape(right)}" - f"{dist:.4f}" - f"{_html_escape(match_type)}" - "" - ) - if rows: - lines.append( - "

      Fingerprint distances

      " - "" - "" - + "".join(rows) - + "" - ) - lines.append("
      ") - return "".join(lines) - - def _render_losers(group: GroupPlan) -> str: - if not group.losers: - return "No losers to consolidate." - rows = [] - for loser in group.losers: - disposition = group.loser_disposition.get(loser, "quarantine") - rewrite = group.playlist_rewrites.get(loser, "n/a") - q = group.track_quality.get(loser, {}) - quality = f"{_html_escape(q.get('container') or '—')} / {_html_escape(q.get('codec') or '—')} ({_html_escape(q.get('bitrate') or '0')} kbps)" - rows.append( - f"{_html_escape(loser)}{quality}{_html_escape(disposition)}" - f"{_html_escape(rewrite)}" - ) - return ( - "" - "" - + "".join(rows) - + "" - ) - - def _render_artwork_section(group: GroupPlan) -> str: - source = group.chosen_artwork_source or {} - source_path = source.get("path") or "None" - source_hash = source.get("hash") or "unknown" - dims = "×".join(str(v) for v in (source.get("width"), source.get("height")) if v) if source.get("width") or source.get("height") else "unknown" - size = f"{source.get('size') or 0} bytes" - lines = [] - if group.artwork: - lines.append("

      Artwork will be applied

        ") - for act in group.artwork: - lines.append( - "
      • " - f"{_html_escape(act.source)} → {_html_escape(act.target)}" - f" ({_html_escape(source_hash)}, {dims}, {size}; {_html_escape(act.reason)})" - "
      • " - ) - lines.append("
      ") - else: - reason = source.get("reason") or group.artwork_status or "Winner artwork unchanged." - lines.append(f"

      Artwork unchanged — {_html_escape(reason)}

      ") - lines.append( - f"

      Artwork source: {_html_escape(source_path)} " - f"(hash={_html_escape(source_hash)}, size={_html_escape(size)}, dimensions={_html_escape(dims)})

      " - ) - if group.artwork_evidence: - lines.append("
        " + "".join(f"
      • {_html_escape(ev)}
      • " for ev in group.artwork_evidence) + "
      ") - return "".join(lines) - - review_required = plan.review_required_groups - - def _basename(path: str) -> str: - base = os.path.basename(path) - return base or path - - lines = [ - "", - "
      ", - "
      ", - "

      Duplicate Consolidation Preview

      ", - f"
      Generated at {plan.generated_at.isoformat()}
      ", - "
      ", - f"
      Total groups {len(plan.groups)}
      ", - "
      ", - "
      ", - f"
      Review required
      {len(review_required)}
      ", - f"
      Global flags
      {len(plan.review_flags)}
      ", - "
      ", - ] - - if plan.review_flags: - lines.append("
      Plan Warnings
        ") - for flag in plan.review_flags: - lines.append(f"
      • {_html_escape(flag)}
      • ") - lines.append("
      ") - - for group in plan.groups: - review_badge = ( - "Review required" - if group.review_flags - else "Ready" - ) - winner_basename = _basename(group.winner_path) - lines.append("
      ") - lines.append("") - lines.append("
      ") - lines.append("
      ") - lines.append(f"Group {group.group_id}") - lines.append( - f"" - f"{_html_escape(winner_basename)}" - ) - lines.append("
      ") - lines.append(review_badge) - lines.append("
      ") - lines.append(f"
      Confidence: {_html_escape(group.group_confidence)}
      ") - lines.append("
      ") - - lines.append("
      ") - lines.append("
      ") - codec_warning = _render_codec_mix_warning(group) - if codec_warning: - lines.append(codec_warning) - lines.append("

      Winner

      ") - lines.append(f"
      {_html_escape(group.winner_path)}
      ") - lines.append(_render_quality(group)) - lines.append("
      ") - lines.append(_render_grouping_evidence(group)) - lines.append("

      Losers

      ") - lines.append(_render_losers(group)) - lines.append("
      ") - lines.append("
      ") - - lines.append("
      ") - lines.append("

      Metadata normalization

      ") - lines.append(_render_metadata(group.metadata_changes)) - lines.append( - "
      " - f"Tag source: {_html_escape(group.tag_source or 'None')} — " - f"{_html_escape(group.tag_source_reason)}
      " - ) - if group.tag_source_evidence: - lines.append("
        " + "".join(f"
      • {_html_escape(ev)}
      • " for ev in group.tag_source_evidence) + "
      ") - lines.append("
      ") - - lines.append("

      Artwork

      ") - lines.append(_render_artwork_section(group)) - lines.append("
      ") - - lines.append( - "

      Playlist impact

      " - f"
      {group.playlist_impact.playlists} playlists, " - f"{group.playlist_impact.entries} entries
      " - ) - - if group.review_flags: - lines.append("

      Review flags

        ") - for flag in group.review_flags: - lines.append(f"
      • {_html_escape(flag)}
      • ") - lines.append("
      ") - - lines.append( - "
      Context & evidence
        " - + f"
      • Album tracks: {len(group.context_summary.get('album', []))}
      • " - + f"
      • Singles: {len(group.context_summary.get('single', []))}
      • " - + f"
      • Unknown: {len(group.context_summary.get('unknown', []))}
      • " - + "
      " - + "
        " - + "".join( - f"
      • {_html_escape(path)}: " - + "; ".join(_html_escape(ev) for ev in group.context_evidence.get(path, [])) - + "
      • " - for path in sorted(group.context_evidence.keys()) - ) - + "
      " - + "

      Artwork evidence

        " - + "".join(f"
      • {_html_escape(ev)}
      • " for ev in group.artwork_evidence) - + "
      " - + "

      Tag source selection

        " - + "".join(f"
      • {_html_escape(ev)}
      • " for ev in group.tag_source_evidence) - + "
      " - + "
      " - ) - lines.append("
      ") - lines.append("
      ") - lines.append("
      ") - - lines.append("") - - with open(output_html_path, "w", encoding="utf-8") as f: - f.write("\n".join(lines)) - return output_html_path - - def export_consolidation_preview(plan: ConsolidationPlan, output_json_path: str) -> str: """Write a JSON audit of the consolidation plan.""" diff --git a/main_gui.py b/main_gui.py index 3cc626d..ffcd6bb 100644 --- a/main_gui.py +++ b/main_gui.py @@ -45,7 +45,6 @@ from duplicate_consolidation import ( build_consolidation_plan, consolidation_plan_from_dict, - render_consolidation_preview, export_consolidation_preview, ) from duplicate_consolidation_executor import ExecutionConfig, execute_consolidation_plan @@ -1476,7 +1475,6 @@ def __init__(self, parent: tk.Widget, library_path: str): self.group_disposition_var = tk.StringVar(value="") self.group_disposition_overrides: dict[str, str] = {} self._selected_group_id: str | None = None - self.preview_html_path: str | None = None self.preview_json_path: str | None = None self.execution_report_path: str | None = None self._plan = None @@ -1532,10 +1530,6 @@ def __init__(self, parent: tk.Widget, library_path: str): ttk.Button(controls, text="Preview", command=self._handle_preview).pack( side="left", padx=(0, 6) ) - self.open_preview_btn = ttk.Button( - controls, text="Open Preview", command=self._open_preview_output, state="disabled" - ) - self.open_preview_btn.pack(side="left", padx=(0, 6)) self.open_report_btn = ttk.Button( controls, text="Open Report", command=self._open_execution_report, state="disabled" ) @@ -1884,20 +1878,13 @@ def _generate_plan(self, write_preview: bool) -> None: docs_dir = os.path.join(path, "Docs") os.makedirs(docs_dir, exist_ok=True) if write_preview: - html_path = os.path.join(docs_dir, "duplicate_preview.html") json_path = os.path.join(docs_dir, "duplicate_preview.json") - render_consolidation_preview(plan, html_path) export_consolidation_preview(plan, json_path) - self.preview_html_path = html_path self.preview_json_path = json_path - self.open_preview_btn.config(state="normal") - self._log_action(f"Preview written to {html_path}") self._log_action(f"Audit JSON written to {json_path}") self._set_status("Preview generated", progress=100) else: - self.preview_html_path = None self.preview_json_path = None - self.open_preview_btn.config(state="disabled") self._set_status("Plan ready", progress=100) self._log_action( @@ -2101,20 +2088,10 @@ def _apply_deletion_mode(self) -> None: self._update_groups_view(self._plan) def _reset_preview_if_needed(self) -> None: - if self.preview_html_path or self.preview_json_path: - self.preview_html_path = None + if self.preview_json_path: self.preview_json_path = None - self.open_preview_btn.config(state="disabled") self._log_action("Preview cleared; generate a new preview to reflect delete settings.") - def _open_preview_output(self) -> None: - self._open_local_html( - self.preview_html_path, - title="Preview", - missing_message="Preview file could not be found. Generate a new preview.", - empty_message="No preview has been generated yet.", - ) - def _open_execution_report(self) -> None: self._open_local_html( self.execution_report_path, diff --git a/tests/test_duplicate_consolidation_preview.py b/tests/test_duplicate_consolidation_preview.py index 771c4a0..d22ad73 100644 --- a/tests/test_duplicate_consolidation_preview.py +++ b/tests/test_duplicate_consolidation_preview.py @@ -4,7 +4,6 @@ from duplicate_consolidation import ( build_consolidation_plan, - render_consolidation_preview, export_consolidation_preview, ) @@ -64,18 +63,12 @@ def test_plan_contains_metadata_and_quality_summary(tmp_path) -> None: assert group.tag_source is not None -def test_preview_render_and_export(tmp_path) -> None: +def test_preview_export(tmp_path) -> None: plan = build_consolidation_plan(_sample_tracks(str(tmp_path))) - html_path = tmp_path / "preview.html" json_path = tmp_path / "preview.json" - render_consolidation_preview(plan, str(html_path)) export_consolidation_preview(plan, str(json_path)) - html = html_path.read_text() - assert "Duplicate Consolidation Preview" in html - assert plan.groups[0].winner_path in html - data = json.loads(json_path.read_text()) assert data["summary"]["groups"] == len(plan.groups) assert data["summary"]["review_required"] == plan.review_required_count From 32d29ed5079c20b1be73d938b362935244866aec Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 28 Dec 2025 18:15:43 -0500 Subject: [PATCH 182/606] Fix HTML report host lookup --- duplicate_consolidation_executor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index 71e344b..be560bb 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -17,6 +17,7 @@ import html import importlib.util import json +import platform import os import shutil import tempfile @@ -1055,7 +1056,7 @@ def _format_metadata_notes(meta: Dict[str, object]) -> str: groups_processed = len(group_lookup) winners_kept = groups_processed generated_at_iso = datetime.datetime.now(datetime.timezone.utc).isoformat() - host_name = os.uname().nodename + host_name = platform.node() or "unknown-host" def _status_badge_class(status: str) -> str: if status == "success": From 4df5184fab36dfd2de248e8cc2e10b65df3dfbb7 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 28 Dec 2025 18:30:38 -0500 Subject: [PATCH 183/606] Add part-of-set subfoldering for albums --- .../indexer_engine/music_indexer_api.py | 56 +++++++++++++++++-- music_indexer_api.py | 56 +++++++++++++++++-- 2 files changed, 104 insertions(+), 8 deletions(-) diff --git a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py index 2426ec5..449e611 100644 --- a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py +++ b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py @@ -124,16 +124,46 @@ def is_repeated(name: str) -> bool: return True return False +def _normalize_part_of_set(raw_value: str | None) -> str | None: + if not raw_value: + return None + text = str(raw_value).strip() + if not text: + return None + first = text.split("/", 1)[0].strip() + if not first: + return None + if first.isdigit(): + return f"Part {int(first)}" + return first + + +def _apply_part_of_set( + base_folder: str, + album: str, + part_of_set: str | None, + decision_log: list[str], +) -> str: + if part_of_set and album and os.path.basename(base_folder) == sanitize(album): + part_folder = sanitize(part_of_set) + decision_log.append( + f" → Part of set '{part_of_set}' detected; placed under subfolder '{part_folder}'" + ) + return os.path.join(base_folder, part_folder) + return base_folder + + def get_tags(path: str): """ - Read basic tags using Mutagen (artist, title, album, year, track, genre). + Read basic tags using Mutagen (artist, title, album, year, track, genre, part_of_set). Return a dict with those fields (or None if missing). """ try: audio = MutagenFile(path, easy=True) if not audio or not audio.tags: return {"artist": None, "title": None, "album": None, - "year": None, "track": None, "genre": None} + "year": None, "track": None, "genre": None, + "part_of_set": None} tags = audio.tags artist = tags.get("artist", [None])[0] title = tags.get("title", [None])[0] @@ -148,11 +178,21 @@ def get_tags(path: str): except Exception: track = None genre = tags.get("genre", [None])[0] + raw_part_of_set = ( + tags.get("partofset", [None])[0] + or tags.get("part_of_set", [None])[0] + or tags.get("part of set", [None])[0] + or tags.get("discnumber", [None])[0] + or tags.get("disc", [None])[0] + ) + part_of_set = _normalize_part_of_set(raw_part_of_set) return {"artist": artist, "title": title, "album": album, - "year": year, "track": track, "genre": genre} + "year": year, "track": track, "genre": genre, + "part_of_set": part_of_set} except Exception: return {"artist": None, "title": None, "album": None, - "year": None, "track": None, "genre": None} + "year": None, "track": None, "genre": None, + "part_of_set": None} def extract_primary_and_collabs(raw_artist: str): """ @@ -519,6 +559,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: year = data["year"] genre = data["genre"] track = data["track"] + part_of_set = data["part_of_set"] # 1) Primary & collabs (preserve original case) primary, collabs = extract_primary_and_collabs(raw_artist) @@ -566,6 +607,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: "year": year, "genre": genre, "track": track, + "part_of_set": part_of_set, "cover_hash": cover_hash, "folder_tags": folder_tags, "missing_core": not data.get("artist") or not data.get("title"), @@ -594,6 +636,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: album = info["album"] or "" year = info["year"] or "Unknown" track = info["track"] + part_of_set = info.get("part_of_set") folders = info["folder_tags"] if info.get("missing_core"): candidate = os.path.join(review_root, os.path.basename(old_path)) @@ -662,6 +705,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: ) artist_folder = os.path.join(MUSIC_ROOT, "By Artist", sanitize(main_artist)) base_folder = os.path.join(artist_folder, sanitize(album)) + base_folder = _apply_part_of_set(base_folder, album, part_of_set, decision_log) # Build filename (skip normal album logic) basename = os.path.basename(old_path) @@ -728,6 +772,8 @@ def _compute(path: str) -> tuple[int | None, str | None]: f"album_count={c2} ≤ 3 → group under '{primary} - Singles'" ) + base_folder = _apply_part_of_set(base_folder, album, part_of_set, decision_log) + # Build filename for all “common” tracks basename = os.path.basename(old_path) if "/" in raw_artist or is_repeated(raw_artist): @@ -752,6 +798,8 @@ def _compute(path: str) -> tuple[int | None, str | None]: leftover.discard(primary) if album: leftover.discard(album) + if part_of_set: + leftover.discard(part_of_set) leftover.discard(year) if genre: leftover.add(genre) diff --git a/music_indexer_api.py b/music_indexer_api.py index 2426ec5..449e611 100644 --- a/music_indexer_api.py +++ b/music_indexer_api.py @@ -124,16 +124,46 @@ def is_repeated(name: str) -> bool: return True return False +def _normalize_part_of_set(raw_value: str | None) -> str | None: + if not raw_value: + return None + text = str(raw_value).strip() + if not text: + return None + first = text.split("/", 1)[0].strip() + if not first: + return None + if first.isdigit(): + return f"Part {int(first)}" + return first + + +def _apply_part_of_set( + base_folder: str, + album: str, + part_of_set: str | None, + decision_log: list[str], +) -> str: + if part_of_set and album and os.path.basename(base_folder) == sanitize(album): + part_folder = sanitize(part_of_set) + decision_log.append( + f" → Part of set '{part_of_set}' detected; placed under subfolder '{part_folder}'" + ) + return os.path.join(base_folder, part_folder) + return base_folder + + def get_tags(path: str): """ - Read basic tags using Mutagen (artist, title, album, year, track, genre). + Read basic tags using Mutagen (artist, title, album, year, track, genre, part_of_set). Return a dict with those fields (or None if missing). """ try: audio = MutagenFile(path, easy=True) if not audio or not audio.tags: return {"artist": None, "title": None, "album": None, - "year": None, "track": None, "genre": None} + "year": None, "track": None, "genre": None, + "part_of_set": None} tags = audio.tags artist = tags.get("artist", [None])[0] title = tags.get("title", [None])[0] @@ -148,11 +178,21 @@ def get_tags(path: str): except Exception: track = None genre = tags.get("genre", [None])[0] + raw_part_of_set = ( + tags.get("partofset", [None])[0] + or tags.get("part_of_set", [None])[0] + or tags.get("part of set", [None])[0] + or tags.get("discnumber", [None])[0] + or tags.get("disc", [None])[0] + ) + part_of_set = _normalize_part_of_set(raw_part_of_set) return {"artist": artist, "title": title, "album": album, - "year": year, "track": track, "genre": genre} + "year": year, "track": track, "genre": genre, + "part_of_set": part_of_set} except Exception: return {"artist": None, "title": None, "album": None, - "year": None, "track": None, "genre": None} + "year": None, "track": None, "genre": None, + "part_of_set": None} def extract_primary_and_collabs(raw_artist: str): """ @@ -519,6 +559,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: year = data["year"] genre = data["genre"] track = data["track"] + part_of_set = data["part_of_set"] # 1) Primary & collabs (preserve original case) primary, collabs = extract_primary_and_collabs(raw_artist) @@ -566,6 +607,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: "year": year, "genre": genre, "track": track, + "part_of_set": part_of_set, "cover_hash": cover_hash, "folder_tags": folder_tags, "missing_core": not data.get("artist") or not data.get("title"), @@ -594,6 +636,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: album = info["album"] or "" year = info["year"] or "Unknown" track = info["track"] + part_of_set = info.get("part_of_set") folders = info["folder_tags"] if info.get("missing_core"): candidate = os.path.join(review_root, os.path.basename(old_path)) @@ -662,6 +705,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: ) artist_folder = os.path.join(MUSIC_ROOT, "By Artist", sanitize(main_artist)) base_folder = os.path.join(artist_folder, sanitize(album)) + base_folder = _apply_part_of_set(base_folder, album, part_of_set, decision_log) # Build filename (skip normal album logic) basename = os.path.basename(old_path) @@ -728,6 +772,8 @@ def _compute(path: str) -> tuple[int | None, str | None]: f"album_count={c2} ≤ 3 → group under '{primary} - Singles'" ) + base_folder = _apply_part_of_set(base_folder, album, part_of_set, decision_log) + # Build filename for all “common” tracks basename = os.path.basename(old_path) if "/" in raw_artist or is_repeated(raw_artist): @@ -752,6 +798,8 @@ def _compute(path: str) -> tuple[int | None, str | None]: leftover.discard(primary) if album: leftover.discard(album) + if part_of_set: + leftover.discard(part_of_set) leftover.discard(year) if genre: leftover.add(genre) From ea26ea22f96ac0a0f8add7553a0eebcb43b29b84 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 28 Dec 2025 18:33:15 -0500 Subject: [PATCH 184/606] Revert "Add part-of-set album subfolder support" --- .../indexer_engine/music_indexer_api.py | 56 ++----------------- music_indexer_api.py | 56 ++----------------- 2 files changed, 8 insertions(+), 104 deletions(-) diff --git a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py index 449e611..2426ec5 100644 --- a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py +++ b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py @@ -124,46 +124,16 @@ def is_repeated(name: str) -> bool: return True return False -def _normalize_part_of_set(raw_value: str | None) -> str | None: - if not raw_value: - return None - text = str(raw_value).strip() - if not text: - return None - first = text.split("/", 1)[0].strip() - if not first: - return None - if first.isdigit(): - return f"Part {int(first)}" - return first - - -def _apply_part_of_set( - base_folder: str, - album: str, - part_of_set: str | None, - decision_log: list[str], -) -> str: - if part_of_set and album and os.path.basename(base_folder) == sanitize(album): - part_folder = sanitize(part_of_set) - decision_log.append( - f" → Part of set '{part_of_set}' detected; placed under subfolder '{part_folder}'" - ) - return os.path.join(base_folder, part_folder) - return base_folder - - def get_tags(path: str): """ - Read basic tags using Mutagen (artist, title, album, year, track, genre, part_of_set). + Read basic tags using Mutagen (artist, title, album, year, track, genre). Return a dict with those fields (or None if missing). """ try: audio = MutagenFile(path, easy=True) if not audio or not audio.tags: return {"artist": None, "title": None, "album": None, - "year": None, "track": None, "genre": None, - "part_of_set": None} + "year": None, "track": None, "genre": None} tags = audio.tags artist = tags.get("artist", [None])[0] title = tags.get("title", [None])[0] @@ -178,21 +148,11 @@ def get_tags(path: str): except Exception: track = None genre = tags.get("genre", [None])[0] - raw_part_of_set = ( - tags.get("partofset", [None])[0] - or tags.get("part_of_set", [None])[0] - or tags.get("part of set", [None])[0] - or tags.get("discnumber", [None])[0] - or tags.get("disc", [None])[0] - ) - part_of_set = _normalize_part_of_set(raw_part_of_set) return {"artist": artist, "title": title, "album": album, - "year": year, "track": track, "genre": genre, - "part_of_set": part_of_set} + "year": year, "track": track, "genre": genre} except Exception: return {"artist": None, "title": None, "album": None, - "year": None, "track": None, "genre": None, - "part_of_set": None} + "year": None, "track": None, "genre": None} def extract_primary_and_collabs(raw_artist: str): """ @@ -559,7 +519,6 @@ def _compute(path: str) -> tuple[int | None, str | None]: year = data["year"] genre = data["genre"] track = data["track"] - part_of_set = data["part_of_set"] # 1) Primary & collabs (preserve original case) primary, collabs = extract_primary_and_collabs(raw_artist) @@ -607,7 +566,6 @@ def _compute(path: str) -> tuple[int | None, str | None]: "year": year, "genre": genre, "track": track, - "part_of_set": part_of_set, "cover_hash": cover_hash, "folder_tags": folder_tags, "missing_core": not data.get("artist") or not data.get("title"), @@ -636,7 +594,6 @@ def _compute(path: str) -> tuple[int | None, str | None]: album = info["album"] or "" year = info["year"] or "Unknown" track = info["track"] - part_of_set = info.get("part_of_set") folders = info["folder_tags"] if info.get("missing_core"): candidate = os.path.join(review_root, os.path.basename(old_path)) @@ -705,7 +662,6 @@ def _compute(path: str) -> tuple[int | None, str | None]: ) artist_folder = os.path.join(MUSIC_ROOT, "By Artist", sanitize(main_artist)) base_folder = os.path.join(artist_folder, sanitize(album)) - base_folder = _apply_part_of_set(base_folder, album, part_of_set, decision_log) # Build filename (skip normal album logic) basename = os.path.basename(old_path) @@ -772,8 +728,6 @@ def _compute(path: str) -> tuple[int | None, str | None]: f"album_count={c2} ≤ 3 → group under '{primary} - Singles'" ) - base_folder = _apply_part_of_set(base_folder, album, part_of_set, decision_log) - # Build filename for all “common” tracks basename = os.path.basename(old_path) if "/" in raw_artist or is_repeated(raw_artist): @@ -798,8 +752,6 @@ def _compute(path: str) -> tuple[int | None, str | None]: leftover.discard(primary) if album: leftover.discard(album) - if part_of_set: - leftover.discard(part_of_set) leftover.discard(year) if genre: leftover.add(genre) diff --git a/music_indexer_api.py b/music_indexer_api.py index 449e611..2426ec5 100644 --- a/music_indexer_api.py +++ b/music_indexer_api.py @@ -124,46 +124,16 @@ def is_repeated(name: str) -> bool: return True return False -def _normalize_part_of_set(raw_value: str | None) -> str | None: - if not raw_value: - return None - text = str(raw_value).strip() - if not text: - return None - first = text.split("/", 1)[0].strip() - if not first: - return None - if first.isdigit(): - return f"Part {int(first)}" - return first - - -def _apply_part_of_set( - base_folder: str, - album: str, - part_of_set: str | None, - decision_log: list[str], -) -> str: - if part_of_set and album and os.path.basename(base_folder) == sanitize(album): - part_folder = sanitize(part_of_set) - decision_log.append( - f" → Part of set '{part_of_set}' detected; placed under subfolder '{part_folder}'" - ) - return os.path.join(base_folder, part_folder) - return base_folder - - def get_tags(path: str): """ - Read basic tags using Mutagen (artist, title, album, year, track, genre, part_of_set). + Read basic tags using Mutagen (artist, title, album, year, track, genre). Return a dict with those fields (or None if missing). """ try: audio = MutagenFile(path, easy=True) if not audio or not audio.tags: return {"artist": None, "title": None, "album": None, - "year": None, "track": None, "genre": None, - "part_of_set": None} + "year": None, "track": None, "genre": None} tags = audio.tags artist = tags.get("artist", [None])[0] title = tags.get("title", [None])[0] @@ -178,21 +148,11 @@ def get_tags(path: str): except Exception: track = None genre = tags.get("genre", [None])[0] - raw_part_of_set = ( - tags.get("partofset", [None])[0] - or tags.get("part_of_set", [None])[0] - or tags.get("part of set", [None])[0] - or tags.get("discnumber", [None])[0] - or tags.get("disc", [None])[0] - ) - part_of_set = _normalize_part_of_set(raw_part_of_set) return {"artist": artist, "title": title, "album": album, - "year": year, "track": track, "genre": genre, - "part_of_set": part_of_set} + "year": year, "track": track, "genre": genre} except Exception: return {"artist": None, "title": None, "album": None, - "year": None, "track": None, "genre": None, - "part_of_set": None} + "year": None, "track": None, "genre": None} def extract_primary_and_collabs(raw_artist: str): """ @@ -559,7 +519,6 @@ def _compute(path: str) -> tuple[int | None, str | None]: year = data["year"] genre = data["genre"] track = data["track"] - part_of_set = data["part_of_set"] # 1) Primary & collabs (preserve original case) primary, collabs = extract_primary_and_collabs(raw_artist) @@ -607,7 +566,6 @@ def _compute(path: str) -> tuple[int | None, str | None]: "year": year, "genre": genre, "track": track, - "part_of_set": part_of_set, "cover_hash": cover_hash, "folder_tags": folder_tags, "missing_core": not data.get("artist") or not data.get("title"), @@ -636,7 +594,6 @@ def _compute(path: str) -> tuple[int | None, str | None]: album = info["album"] or "" year = info["year"] or "Unknown" track = info["track"] - part_of_set = info.get("part_of_set") folders = info["folder_tags"] if info.get("missing_core"): candidate = os.path.join(review_root, os.path.basename(old_path)) @@ -705,7 +662,6 @@ def _compute(path: str) -> tuple[int | None, str | None]: ) artist_folder = os.path.join(MUSIC_ROOT, "By Artist", sanitize(main_artist)) base_folder = os.path.join(artist_folder, sanitize(album)) - base_folder = _apply_part_of_set(base_folder, album, part_of_set, decision_log) # Build filename (skip normal album logic) basename = os.path.basename(old_path) @@ -772,8 +728,6 @@ def _compute(path: str) -> tuple[int | None, str | None]: f"album_count={c2} ≤ 3 → group under '{primary} - Singles'" ) - base_folder = _apply_part_of_set(base_folder, album, part_of_set, decision_log) - # Build filename for all “common” tracks basename = os.path.basename(old_path) if "/" in raw_artist or is_repeated(raw_artist): @@ -798,8 +752,6 @@ def _compute(path: str) -> tuple[int | None, str | None]: leftover.discard(primary) if album: leftover.discard(album) - if part_of_set: - leftover.discard(part_of_set) leftover.discard(year) if genre: leftover.add(genre) From c25354a4d0f1839c7b811dc84117bf48ebd177c8 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 28 Dec 2025 18:38:32 -0500 Subject: [PATCH 185/606] Normalize fallback artist parsing --- .../indexer_engine/music_indexer_api.py | 77 +++++++++++++++---- music_indexer_api.py | 77 +++++++++++++++---- 2 files changed, 126 insertions(+), 28 deletions(-) diff --git a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py index 2426ec5..840c102 100644 --- a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py +++ b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py @@ -75,15 +75,8 @@ def progress_callback(_c, _t, _m, _p): # Normalize artist identically to Phase 4 raw = (tags.get("artist") or "").strip() - if not raw: - name_only = os.path.splitext(fname)[0] - if "_" in name_only: - raw = name_only.split("_", 1)[0] - elif " - " in name_only: - raw = name_only.split(" - ", 1)[0] - else: - raw = name_only + raw = _fallback_artist_from_filename(fname) primary, _ = extract_primary_and_collabs(raw) p_lower = primary.lower() @@ -96,6 +89,14 @@ def sanitize(name: str) -> str: invalid = r'<>:"/\\|?*' return "".join(c for c in (name or "Unknown") if c not in invalid).strip() or "Unknown" +def _fallback_artist_from_filename(filename: str) -> str: + name_only = os.path.splitext(os.path.basename(filename))[0] + if "_" in name_only: + return name_only.split("_", 1)[0] + if " - " in name_only: + return name_only.split(" - ", 1)[0] + return name_only + def collapse_repeats(name: str) -> str: """ If name is something like 'DROELOEDROELOE', collapse to 'DROELOE'. @@ -124,16 +125,46 @@ def is_repeated(name: str) -> bool: return True return False +def _normalize_part_of_set(raw_value: str | None) -> str | None: + if not raw_value: + return None + text = str(raw_value).strip() + if not text: + return None + first = text.split("/", 1)[0].strip() + if not first: + return None + if first.isdigit(): + return f"Part {int(first)}" + return first + + +def _apply_part_of_set( + base_folder: str, + album: str, + part_of_set: str | None, + decision_log: list[str], +) -> str: + if part_of_set and album and os.path.basename(base_folder) == sanitize(album): + part_folder = sanitize(part_of_set) + decision_log.append( + f" → Part of set '{part_of_set}' detected; placed under subfolder '{part_folder}'" + ) + return os.path.join(base_folder, part_folder) + return base_folder + + def get_tags(path: str): """ - Read basic tags using Mutagen (artist, title, album, year, track, genre). + Read basic tags using Mutagen (artist, title, album, year, track, genre, part_of_set). Return a dict with those fields (or None if missing). """ try: audio = MutagenFile(path, easy=True) if not audio or not audio.tags: return {"artist": None, "title": None, "album": None, - "year": None, "track": None, "genre": None} + "year": None, "track": None, "genre": None, + "part_of_set": None} tags = audio.tags artist = tags.get("artist", [None])[0] title = tags.get("title", [None])[0] @@ -148,11 +179,21 @@ def get_tags(path: str): except Exception: track = None genre = tags.get("genre", [None])[0] + raw_part_of_set = ( + tags.get("partofset", [None])[0] + or tags.get("part_of_set", [None])[0] + or tags.get("part of set", [None])[0] + or tags.get("discnumber", [None])[0] + or tags.get("disc", [None])[0] + ) + part_of_set = _normalize_part_of_set(raw_part_of_set) return {"artist": artist, "title": title, "album": album, - "year": year, "track": track, "genre": genre} + "year": year, "track": track, "genre": genre, + "part_of_set": part_of_set} except Exception: return {"artist": None, "title": None, "album": None, - "year": None, "track": None, "genre": None} + "year": None, "track": None, "genre": None, + "part_of_set": None} def extract_primary_and_collabs(raw_artist: str): """ @@ -379,7 +420,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: if idx % 50 == 0 or idx == total_audio: log_callback(f" • Processing file {idx}/{total_audio} for dedupe") data = get_tags(fullpath) - raw_artist = data["artist"] or os.path.splitext(os.path.basename(fullpath))[0] + raw_artist = data["artist"] or _fallback_artist_from_filename(fullpath) raw_artist = collapse_repeats(raw_artist) title = data["title"] or os.path.splitext(os.path.basename(fullpath))[0] album = data["album"] or "" @@ -512,13 +553,14 @@ def _compute(path: str) -> tuple[int | None, str | None]: progress_callback(idx, total_kept, f"Metadata {os.path.relpath(fullpath, root_path)}", "B") data = get_tags(fullpath) - raw_artist = data["artist"] or os.path.splitext(os.path.basename(fullpath))[0] + raw_artist = data["artist"] or _fallback_artist_from_filename(fullpath) raw_artist = collapse_repeats(raw_artist) title = data["title"] or os.path.splitext(os.path.basename(fullpath))[0] album = data["album"] year = data["year"] genre = data["genre"] track = data["track"] + part_of_set = data["part_of_set"] # 1) Primary & collabs (preserve original case) primary, collabs = extract_primary_and_collabs(raw_artist) @@ -566,6 +608,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: "year": year, "genre": genre, "track": track, + "part_of_set": part_of_set, "cover_hash": cover_hash, "folder_tags": folder_tags, "missing_core": not data.get("artist") or not data.get("title"), @@ -594,6 +637,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: album = info["album"] or "" year = info["year"] or "Unknown" track = info["track"] + part_of_set = info.get("part_of_set") folders = info["folder_tags"] if info.get("missing_core"): candidate = os.path.join(review_root, os.path.basename(old_path)) @@ -662,6 +706,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: ) artist_folder = os.path.join(MUSIC_ROOT, "By Artist", sanitize(main_artist)) base_folder = os.path.join(artist_folder, sanitize(album)) + base_folder = _apply_part_of_set(base_folder, album, part_of_set, decision_log) # Build filename (skip normal album logic) basename = os.path.basename(old_path) @@ -728,6 +773,8 @@ def _compute(path: str) -> tuple[int | None, str | None]: f"album_count={c2} ≤ 3 → group under '{primary} - Singles'" ) + base_folder = _apply_part_of_set(base_folder, album, part_of_set, decision_log) + # Build filename for all “common” tracks basename = os.path.basename(old_path) if "/" in raw_artist or is_repeated(raw_artist): @@ -752,6 +799,8 @@ def _compute(path: str) -> tuple[int | None, str | None]: leftover.discard(primary) if album: leftover.discard(album) + if part_of_set: + leftover.discard(part_of_set) leftover.discard(year) if genre: leftover.add(genre) diff --git a/music_indexer_api.py b/music_indexer_api.py index 2426ec5..840c102 100644 --- a/music_indexer_api.py +++ b/music_indexer_api.py @@ -75,15 +75,8 @@ def progress_callback(_c, _t, _m, _p): # Normalize artist identically to Phase 4 raw = (tags.get("artist") or "").strip() - if not raw: - name_only = os.path.splitext(fname)[0] - if "_" in name_only: - raw = name_only.split("_", 1)[0] - elif " - " in name_only: - raw = name_only.split(" - ", 1)[0] - else: - raw = name_only + raw = _fallback_artist_from_filename(fname) primary, _ = extract_primary_and_collabs(raw) p_lower = primary.lower() @@ -96,6 +89,14 @@ def sanitize(name: str) -> str: invalid = r'<>:"/\\|?*' return "".join(c for c in (name or "Unknown") if c not in invalid).strip() or "Unknown" +def _fallback_artist_from_filename(filename: str) -> str: + name_only = os.path.splitext(os.path.basename(filename))[0] + if "_" in name_only: + return name_only.split("_", 1)[0] + if " - " in name_only: + return name_only.split(" - ", 1)[0] + return name_only + def collapse_repeats(name: str) -> str: """ If name is something like 'DROELOEDROELOE', collapse to 'DROELOE'. @@ -124,16 +125,46 @@ def is_repeated(name: str) -> bool: return True return False +def _normalize_part_of_set(raw_value: str | None) -> str | None: + if not raw_value: + return None + text = str(raw_value).strip() + if not text: + return None + first = text.split("/", 1)[0].strip() + if not first: + return None + if first.isdigit(): + return f"Part {int(first)}" + return first + + +def _apply_part_of_set( + base_folder: str, + album: str, + part_of_set: str | None, + decision_log: list[str], +) -> str: + if part_of_set and album and os.path.basename(base_folder) == sanitize(album): + part_folder = sanitize(part_of_set) + decision_log.append( + f" → Part of set '{part_of_set}' detected; placed under subfolder '{part_folder}'" + ) + return os.path.join(base_folder, part_folder) + return base_folder + + def get_tags(path: str): """ - Read basic tags using Mutagen (artist, title, album, year, track, genre). + Read basic tags using Mutagen (artist, title, album, year, track, genre, part_of_set). Return a dict with those fields (or None if missing). """ try: audio = MutagenFile(path, easy=True) if not audio or not audio.tags: return {"artist": None, "title": None, "album": None, - "year": None, "track": None, "genre": None} + "year": None, "track": None, "genre": None, + "part_of_set": None} tags = audio.tags artist = tags.get("artist", [None])[0] title = tags.get("title", [None])[0] @@ -148,11 +179,21 @@ def get_tags(path: str): except Exception: track = None genre = tags.get("genre", [None])[0] + raw_part_of_set = ( + tags.get("partofset", [None])[0] + or tags.get("part_of_set", [None])[0] + or tags.get("part of set", [None])[0] + or tags.get("discnumber", [None])[0] + or tags.get("disc", [None])[0] + ) + part_of_set = _normalize_part_of_set(raw_part_of_set) return {"artist": artist, "title": title, "album": album, - "year": year, "track": track, "genre": genre} + "year": year, "track": track, "genre": genre, + "part_of_set": part_of_set} except Exception: return {"artist": None, "title": None, "album": None, - "year": None, "track": None, "genre": None} + "year": None, "track": None, "genre": None, + "part_of_set": None} def extract_primary_and_collabs(raw_artist: str): """ @@ -379,7 +420,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: if idx % 50 == 0 or idx == total_audio: log_callback(f" • Processing file {idx}/{total_audio} for dedupe") data = get_tags(fullpath) - raw_artist = data["artist"] or os.path.splitext(os.path.basename(fullpath))[0] + raw_artist = data["artist"] or _fallback_artist_from_filename(fullpath) raw_artist = collapse_repeats(raw_artist) title = data["title"] or os.path.splitext(os.path.basename(fullpath))[0] album = data["album"] or "" @@ -512,13 +553,14 @@ def _compute(path: str) -> tuple[int | None, str | None]: progress_callback(idx, total_kept, f"Metadata {os.path.relpath(fullpath, root_path)}", "B") data = get_tags(fullpath) - raw_artist = data["artist"] or os.path.splitext(os.path.basename(fullpath))[0] + raw_artist = data["artist"] or _fallback_artist_from_filename(fullpath) raw_artist = collapse_repeats(raw_artist) title = data["title"] or os.path.splitext(os.path.basename(fullpath))[0] album = data["album"] year = data["year"] genre = data["genre"] track = data["track"] + part_of_set = data["part_of_set"] # 1) Primary & collabs (preserve original case) primary, collabs = extract_primary_and_collabs(raw_artist) @@ -566,6 +608,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: "year": year, "genre": genre, "track": track, + "part_of_set": part_of_set, "cover_hash": cover_hash, "folder_tags": folder_tags, "missing_core": not data.get("artist") or not data.get("title"), @@ -594,6 +637,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: album = info["album"] or "" year = info["year"] or "Unknown" track = info["track"] + part_of_set = info.get("part_of_set") folders = info["folder_tags"] if info.get("missing_core"): candidate = os.path.join(review_root, os.path.basename(old_path)) @@ -662,6 +706,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: ) artist_folder = os.path.join(MUSIC_ROOT, "By Artist", sanitize(main_artist)) base_folder = os.path.join(artist_folder, sanitize(album)) + base_folder = _apply_part_of_set(base_folder, album, part_of_set, decision_log) # Build filename (skip normal album logic) basename = os.path.basename(old_path) @@ -728,6 +773,8 @@ def _compute(path: str) -> tuple[int | None, str | None]: f"album_count={c2} ≤ 3 → group under '{primary} - Singles'" ) + base_folder = _apply_part_of_set(base_folder, album, part_of_set, decision_log) + # Build filename for all “common” tracks basename = os.path.basename(old_path) if "/" in raw_artist or is_repeated(raw_artist): @@ -752,6 +799,8 @@ def _compute(path: str) -> tuple[int | None, str | None]: leftover.discard(primary) if album: leftover.discard(album) + if part_of_set: + leftover.discard(part_of_set) leftover.discard(year) if genre: leftover.add(genre) From a41ba61874b5d55c014d42876c65ad66eadbe34c Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 28 Dec 2025 18:49:38 -0500 Subject: [PATCH 186/606] Normalize artist and album display names --- .../indexer_engine/music_indexer_api.py | 123 ++++++++++++++---- music_indexer_api.py | 123 ++++++++++++++---- 2 files changed, 194 insertions(+), 52 deletions(-) diff --git a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py index 2426ec5..3c79d81 100644 --- a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py +++ b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py @@ -75,15 +75,8 @@ def progress_callback(_c, _t, _m, _p): # Normalize artist identically to Phase 4 raw = (tags.get("artist") or "").strip() - if not raw: - name_only = os.path.splitext(fname)[0] - if "_" in name_only: - raw = name_only.split("_", 1)[0] - elif " - " in name_only: - raw = name_only.split(" - ", 1)[0] - else: - raw = name_only + raw = _fallback_artist_from_filename(fname) primary, _ = extract_primary_and_collabs(raw) p_lower = primary.lower() @@ -96,6 +89,14 @@ def sanitize(name: str) -> str: invalid = r'<>:"/\\|?*' return "".join(c for c in (name or "Unknown") if c not in invalid).strip() or "Unknown" +def _fallback_artist_from_filename(filename: str) -> str: + name_only = os.path.splitext(os.path.basename(filename))[0] + if "_" in name_only: + return name_only.split("_", 1)[0] + if " - " in name_only: + return name_only.split(" - ", 1)[0] + return name_only + def collapse_repeats(name: str) -> str: """ If name is something like 'DROELOEDROELOE', collapse to 'DROELOE'. @@ -124,16 +125,46 @@ def is_repeated(name: str) -> bool: return True return False +def _normalize_part_of_set(raw_value: str | None) -> str | None: + if not raw_value: + return None + text = str(raw_value).strip() + if not text: + return None + first = text.split("/", 1)[0].strip() + if not first: + return None + if first.isdigit(): + return f"Part {int(first)}" + return first + + +def _apply_part_of_set( + base_folder: str, + album_folder: str, + part_of_set: str | None, + decision_log: list[str], +) -> str: + if part_of_set and album_folder and os.path.basename(base_folder) == sanitize(album_folder): + part_folder = sanitize(part_of_set) + decision_log.append( + f" → Part of set '{part_of_set}' detected; placed under subfolder '{part_folder}'" + ) + return os.path.join(base_folder, part_folder) + return base_folder + + def get_tags(path: str): """ - Read basic tags using Mutagen (artist, title, album, year, track, genre). + Read basic tags using Mutagen (artist, title, album, year, track, genre, part_of_set). Return a dict with those fields (or None if missing). """ try: audio = MutagenFile(path, easy=True) if not audio or not audio.tags: return {"artist": None, "title": None, "album": None, - "year": None, "track": None, "genre": None} + "year": None, "track": None, "genre": None, + "part_of_set": None} tags = audio.tags artist = tags.get("artist", [None])[0] title = tags.get("title", [None])[0] @@ -148,11 +179,21 @@ def get_tags(path: str): except Exception: track = None genre = tags.get("genre", [None])[0] + raw_part_of_set = ( + tags.get("partofset", [None])[0] + or tags.get("part_of_set", [None])[0] + or tags.get("part of set", [None])[0] + or tags.get("discnumber", [None])[0] + or tags.get("disc", [None])[0] + ) + part_of_set = _normalize_part_of_set(raw_part_of_set) return {"artist": artist, "title": title, "album": album, - "year": year, "track": track, "genre": genre} + "year": year, "track": track, "genre": genre, + "part_of_set": part_of_set} except Exception: return {"artist": None, "title": None, "album": None, - "year": None, "track": None, "genre": None} + "year": None, "track": None, "genre": None, + "part_of_set": None} def extract_primary_and_collabs(raw_artist: str): """ @@ -379,7 +420,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: if idx % 50 == 0 or idx == total_audio: log_callback(f" • Processing file {idx}/{total_audio} for dedupe") data = get_tags(fullpath) - raw_artist = data["artist"] or os.path.splitext(os.path.basename(fullpath))[0] + raw_artist = data["artist"] or _fallback_artist_from_filename(fullpath) raw_artist = collapse_repeats(raw_artist) title = data["title"] or os.path.splitext(os.path.basename(fullpath))[0] album = data["album"] or "" @@ -503,6 +544,8 @@ def _compute(path: str) -> tuple[int | None, str | None]: album_counts = defaultdict(int) remix_counts = defaultdict(int) cover_counts = defaultdict(int) + primary_display_counts: Dict[str, Dict[str, int]] = defaultdict(lambda: defaultdict(int)) + album_display_counts: Dict[tuple[str, str], Dict[str, int]] = defaultdict(lambda: defaultdict(int)) total_kept = len(kept_files) progress_callback(0, total_kept, "Reading metadata", "B") @@ -512,18 +555,23 @@ def _compute(path: str) -> tuple[int | None, str | None]: progress_callback(idx, total_kept, f"Metadata {os.path.relpath(fullpath, root_path)}", "B") data = get_tags(fullpath) - raw_artist = data["artist"] or os.path.splitext(os.path.basename(fullpath))[0] + raw_artist = data["artist"] or _fallback_artist_from_filename(fullpath) raw_artist = collapse_repeats(raw_artist) title = data["title"] or os.path.splitext(os.path.basename(fullpath))[0] album = data["album"] year = data["year"] genre = data["genre"] track = data["track"] + part_of_set = data["part_of_set"] # 1) Primary & collabs (preserve original case) primary, collabs = extract_primary_and_collabs(raw_artist) p_lower = primary.lower() + primary_display_counts[p_lower][primary] += 1 + if album: + album_display_counts[(p_lower, album.lower())][album] += 1 + # 2) album_counts for genuine (non-remix) tracks under each (artist, album) if album and "remix" not in title.lower() and album.strip().lower() != title.strip().lower(): album_counts[(p_lower, album.lower())] += 1 @@ -566,12 +614,21 @@ def _compute(path: str) -> tuple[int | None, str | None]: "year": year, "genre": genre, "track": track, + "part_of_set": part_of_set, "cover_hash": cover_hash, "folder_tags": folder_tags, "missing_core": not data.get("artist") or not data.get("title"), } log_callback(f" → Collected metadata for {total_kept} files.") + primary_display_names = { + key: max(values, key=lambda name: (values[name], len(name))) + for key, values in primary_display_counts.items() + } + album_display_names = { + key: max(values, key=lambda name: (values[name], len(name))) + for key, values in album_display_counts.items() + } # ─── Phase 4: Determine destination for each file (with logging) ───────────── log_callback("4/6: Determining destination paths for each file…") @@ -594,6 +651,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: album = info["album"] or "" year = info["year"] or "Unknown" track = info["track"] + part_of_set = info.get("part_of_set") folders = info["folder_tags"] if info.get("missing_core"): candidate = os.path.join(review_root, os.path.basename(old_path)) @@ -656,12 +714,15 @@ def _compute(path: str) -> tuple[int | None, str | None]: # Promote primary to remixer (first part of raw_artist before “/”) main_artist = raw_artist.split("/", 1)[0] p_lower = main_artist.lower() + main_artist_display = primary_display_names.get(p_lower, main_artist) decision_log.append( f" → Enough remixes ({rcount} ≥ {REMIX_FOLDER_THRESHOLD}) " - f"AND count_now ({count_now}) ≥ {COMMON_ARTIST_THRESHOLD}; force primary = '{main_artist}'" + f"AND count_now ({count_now}) ≥ {COMMON_ARTIST_THRESHOLD}; force primary = '{main_artist_display}'" ) - artist_folder = os.path.join(MUSIC_ROOT, "By Artist", sanitize(main_artist)) - base_folder = os.path.join(artist_folder, sanitize(album)) + artist_folder = os.path.join(MUSIC_ROOT, "By Artist", sanitize(main_artist_display)) + album_display = album_display_names.get((p_lower, album.lower()), album) + base_folder = os.path.join(artist_folder, sanitize(album_display)) + base_folder = _apply_part_of_set(base_folder, album_display, part_of_set, decision_log) # Build filename (skip normal album logic) basename = os.path.basename(old_path) @@ -679,7 +740,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: new_path = os.path.join(base_folder, new_filename) moves[old_path] = new_path decision_log.append( - f" → (Remixes folder) placed under By Artist/{main_artist}/{album} " + f" → (Remixes folder) placed under By Artist/{main_artist_display}/{album_display} " f"→ Final: '{os.path.relpath(new_path, MUSIC_ROOT)}'" ) continue @@ -697,37 +758,43 @@ def _compute(path: str) -> tuple[int | None, str | None]: decision_log.append(f" Candidates: {all_candidates}, counts: {counts}") primary = best_artist p_lower = primary.lower() - decision_log.append(f" → After ranking, primary = '{primary}'") + primary_display = primary_display_names.get(p_lower, primary) + decision_log.append(f" → After ranking, primary = '{primary_display}'") # Build “By Artist/” base path - artist_folder = os.path.join(MUSIC_ROOT, "By Artist", sanitize(primary)) + artist_folder = os.path.join(MUSIC_ROOT, "By Artist", sanitize(primary_display)) # ─── 4.4) COMMON ARTIST: decide between “Album” vs. “Singles” ─────────────── # (At this point, count_now ≥ COMMON_ARTIST_THRESHOLD is guaranteed.) if not album or album.strip().lower() == title.strip().lower(): # No album tag (or album == title) → put into “ - Singles” - base_folder = os.path.join(artist_folder, f"{sanitize(primary)} - Singles") + base_folder = os.path.join(artist_folder, f"{sanitize(primary_display)} - Singles") decision_log.append( f" Count {count_now} ≥ {COMMON_ARTIST_THRESHOLD}, no album or album=title " - f"→ group under '{primary} - Singles'" + f"→ group under '{primary_display} - Singles'" ) else: c2 = album_counts.get((p_lower, album.lower()), 0) if c2 > 3: # Enough tracks in this album → put into “/” - base_folder = os.path.join(artist_folder, sanitize(album)) + album_display = album_display_names.get((p_lower, album.lower()), album) + base_folder = os.path.join(artist_folder, sanitize(album_display)) decision_log.append( f" Count {count_now} ≥ {COMMON_ARTIST_THRESHOLD}, album_count={c2} > 3 " - f"→ group under Album '{album}'" + f"→ group under Album '{album_display}'" ) else: # Few tracks in album → treat as a “Singles” release - base_folder = os.path.join(artist_folder, f"{sanitize(primary)} - Singles") + base_folder = os.path.join(artist_folder, f"{sanitize(primary_display)} - Singles") decision_log.append( f" Count {count_now} ≥ {COMMON_ARTIST_THRESHOLD}, but " - f"album_count={c2} ≤ 3 → group under '{primary} - Singles'" + f"album_count={c2} ≤ 3 → group under '{primary_display} - Singles'" ) + if album: + album_display = album_display_names.get((p_lower, album.lower()), album) + base_folder = _apply_part_of_set(base_folder, album_display, part_of_set, decision_log) + # Build filename for all “common” tracks basename = os.path.basename(old_path) if "/" in raw_artist or is_repeated(raw_artist): @@ -750,8 +817,12 @@ def _compute(path: str) -> tuple[int | None, str | None]: # ─── 4.4) Build “leftover_tags” for HTML preview (optional) ─────────────── leftover = set(folders) leftover.discard(primary) + leftover.discard(primary_display) if album: leftover.discard(album) + leftover.discard(album_display_names.get((p_lower, album.lower()), album)) + if part_of_set: + leftover.discard(part_of_set) leftover.discard(year) if genre: leftover.add(genre) diff --git a/music_indexer_api.py b/music_indexer_api.py index 2426ec5..3c79d81 100644 --- a/music_indexer_api.py +++ b/music_indexer_api.py @@ -75,15 +75,8 @@ def progress_callback(_c, _t, _m, _p): # Normalize artist identically to Phase 4 raw = (tags.get("artist") or "").strip() - if not raw: - name_only = os.path.splitext(fname)[0] - if "_" in name_only: - raw = name_only.split("_", 1)[0] - elif " - " in name_only: - raw = name_only.split(" - ", 1)[0] - else: - raw = name_only + raw = _fallback_artist_from_filename(fname) primary, _ = extract_primary_and_collabs(raw) p_lower = primary.lower() @@ -96,6 +89,14 @@ def sanitize(name: str) -> str: invalid = r'<>:"/\\|?*' return "".join(c for c in (name or "Unknown") if c not in invalid).strip() or "Unknown" +def _fallback_artist_from_filename(filename: str) -> str: + name_only = os.path.splitext(os.path.basename(filename))[0] + if "_" in name_only: + return name_only.split("_", 1)[0] + if " - " in name_only: + return name_only.split(" - ", 1)[0] + return name_only + def collapse_repeats(name: str) -> str: """ If name is something like 'DROELOEDROELOE', collapse to 'DROELOE'. @@ -124,16 +125,46 @@ def is_repeated(name: str) -> bool: return True return False +def _normalize_part_of_set(raw_value: str | None) -> str | None: + if not raw_value: + return None + text = str(raw_value).strip() + if not text: + return None + first = text.split("/", 1)[0].strip() + if not first: + return None + if first.isdigit(): + return f"Part {int(first)}" + return first + + +def _apply_part_of_set( + base_folder: str, + album_folder: str, + part_of_set: str | None, + decision_log: list[str], +) -> str: + if part_of_set and album_folder and os.path.basename(base_folder) == sanitize(album_folder): + part_folder = sanitize(part_of_set) + decision_log.append( + f" → Part of set '{part_of_set}' detected; placed under subfolder '{part_folder}'" + ) + return os.path.join(base_folder, part_folder) + return base_folder + + def get_tags(path: str): """ - Read basic tags using Mutagen (artist, title, album, year, track, genre). + Read basic tags using Mutagen (artist, title, album, year, track, genre, part_of_set). Return a dict with those fields (or None if missing). """ try: audio = MutagenFile(path, easy=True) if not audio or not audio.tags: return {"artist": None, "title": None, "album": None, - "year": None, "track": None, "genre": None} + "year": None, "track": None, "genre": None, + "part_of_set": None} tags = audio.tags artist = tags.get("artist", [None])[0] title = tags.get("title", [None])[0] @@ -148,11 +179,21 @@ def get_tags(path: str): except Exception: track = None genre = tags.get("genre", [None])[0] + raw_part_of_set = ( + tags.get("partofset", [None])[0] + or tags.get("part_of_set", [None])[0] + or tags.get("part of set", [None])[0] + or tags.get("discnumber", [None])[0] + or tags.get("disc", [None])[0] + ) + part_of_set = _normalize_part_of_set(raw_part_of_set) return {"artist": artist, "title": title, "album": album, - "year": year, "track": track, "genre": genre} + "year": year, "track": track, "genre": genre, + "part_of_set": part_of_set} except Exception: return {"artist": None, "title": None, "album": None, - "year": None, "track": None, "genre": None} + "year": None, "track": None, "genre": None, + "part_of_set": None} def extract_primary_and_collabs(raw_artist: str): """ @@ -379,7 +420,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: if idx % 50 == 0 or idx == total_audio: log_callback(f" • Processing file {idx}/{total_audio} for dedupe") data = get_tags(fullpath) - raw_artist = data["artist"] or os.path.splitext(os.path.basename(fullpath))[0] + raw_artist = data["artist"] or _fallback_artist_from_filename(fullpath) raw_artist = collapse_repeats(raw_artist) title = data["title"] or os.path.splitext(os.path.basename(fullpath))[0] album = data["album"] or "" @@ -503,6 +544,8 @@ def _compute(path: str) -> tuple[int | None, str | None]: album_counts = defaultdict(int) remix_counts = defaultdict(int) cover_counts = defaultdict(int) + primary_display_counts: Dict[str, Dict[str, int]] = defaultdict(lambda: defaultdict(int)) + album_display_counts: Dict[tuple[str, str], Dict[str, int]] = defaultdict(lambda: defaultdict(int)) total_kept = len(kept_files) progress_callback(0, total_kept, "Reading metadata", "B") @@ -512,18 +555,23 @@ def _compute(path: str) -> tuple[int | None, str | None]: progress_callback(idx, total_kept, f"Metadata {os.path.relpath(fullpath, root_path)}", "B") data = get_tags(fullpath) - raw_artist = data["artist"] or os.path.splitext(os.path.basename(fullpath))[0] + raw_artist = data["artist"] or _fallback_artist_from_filename(fullpath) raw_artist = collapse_repeats(raw_artist) title = data["title"] or os.path.splitext(os.path.basename(fullpath))[0] album = data["album"] year = data["year"] genre = data["genre"] track = data["track"] + part_of_set = data["part_of_set"] # 1) Primary & collabs (preserve original case) primary, collabs = extract_primary_and_collabs(raw_artist) p_lower = primary.lower() + primary_display_counts[p_lower][primary] += 1 + if album: + album_display_counts[(p_lower, album.lower())][album] += 1 + # 2) album_counts for genuine (non-remix) tracks under each (artist, album) if album and "remix" not in title.lower() and album.strip().lower() != title.strip().lower(): album_counts[(p_lower, album.lower())] += 1 @@ -566,12 +614,21 @@ def _compute(path: str) -> tuple[int | None, str | None]: "year": year, "genre": genre, "track": track, + "part_of_set": part_of_set, "cover_hash": cover_hash, "folder_tags": folder_tags, "missing_core": not data.get("artist") or not data.get("title"), } log_callback(f" → Collected metadata for {total_kept} files.") + primary_display_names = { + key: max(values, key=lambda name: (values[name], len(name))) + for key, values in primary_display_counts.items() + } + album_display_names = { + key: max(values, key=lambda name: (values[name], len(name))) + for key, values in album_display_counts.items() + } # ─── Phase 4: Determine destination for each file (with logging) ───────────── log_callback("4/6: Determining destination paths for each file…") @@ -594,6 +651,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: album = info["album"] or "" year = info["year"] or "Unknown" track = info["track"] + part_of_set = info.get("part_of_set") folders = info["folder_tags"] if info.get("missing_core"): candidate = os.path.join(review_root, os.path.basename(old_path)) @@ -656,12 +714,15 @@ def _compute(path: str) -> tuple[int | None, str | None]: # Promote primary to remixer (first part of raw_artist before “/”) main_artist = raw_artist.split("/", 1)[0] p_lower = main_artist.lower() + main_artist_display = primary_display_names.get(p_lower, main_artist) decision_log.append( f" → Enough remixes ({rcount} ≥ {REMIX_FOLDER_THRESHOLD}) " - f"AND count_now ({count_now}) ≥ {COMMON_ARTIST_THRESHOLD}; force primary = '{main_artist}'" + f"AND count_now ({count_now}) ≥ {COMMON_ARTIST_THRESHOLD}; force primary = '{main_artist_display}'" ) - artist_folder = os.path.join(MUSIC_ROOT, "By Artist", sanitize(main_artist)) - base_folder = os.path.join(artist_folder, sanitize(album)) + artist_folder = os.path.join(MUSIC_ROOT, "By Artist", sanitize(main_artist_display)) + album_display = album_display_names.get((p_lower, album.lower()), album) + base_folder = os.path.join(artist_folder, sanitize(album_display)) + base_folder = _apply_part_of_set(base_folder, album_display, part_of_set, decision_log) # Build filename (skip normal album logic) basename = os.path.basename(old_path) @@ -679,7 +740,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: new_path = os.path.join(base_folder, new_filename) moves[old_path] = new_path decision_log.append( - f" → (Remixes folder) placed under By Artist/{main_artist}/{album} " + f" → (Remixes folder) placed under By Artist/{main_artist_display}/{album_display} " f"→ Final: '{os.path.relpath(new_path, MUSIC_ROOT)}'" ) continue @@ -697,37 +758,43 @@ def _compute(path: str) -> tuple[int | None, str | None]: decision_log.append(f" Candidates: {all_candidates}, counts: {counts}") primary = best_artist p_lower = primary.lower() - decision_log.append(f" → After ranking, primary = '{primary}'") + primary_display = primary_display_names.get(p_lower, primary) + decision_log.append(f" → After ranking, primary = '{primary_display}'") # Build “By Artist/” base path - artist_folder = os.path.join(MUSIC_ROOT, "By Artist", sanitize(primary)) + artist_folder = os.path.join(MUSIC_ROOT, "By Artist", sanitize(primary_display)) # ─── 4.4) COMMON ARTIST: decide between “Album” vs. “Singles” ─────────────── # (At this point, count_now ≥ COMMON_ARTIST_THRESHOLD is guaranteed.) if not album or album.strip().lower() == title.strip().lower(): # No album tag (or album == title) → put into “ - Singles” - base_folder = os.path.join(artist_folder, f"{sanitize(primary)} - Singles") + base_folder = os.path.join(artist_folder, f"{sanitize(primary_display)} - Singles") decision_log.append( f" Count {count_now} ≥ {COMMON_ARTIST_THRESHOLD}, no album or album=title " - f"→ group under '{primary} - Singles'" + f"→ group under '{primary_display} - Singles'" ) else: c2 = album_counts.get((p_lower, album.lower()), 0) if c2 > 3: # Enough tracks in this album → put into “/” - base_folder = os.path.join(artist_folder, sanitize(album)) + album_display = album_display_names.get((p_lower, album.lower()), album) + base_folder = os.path.join(artist_folder, sanitize(album_display)) decision_log.append( f" Count {count_now} ≥ {COMMON_ARTIST_THRESHOLD}, album_count={c2} > 3 " - f"→ group under Album '{album}'" + f"→ group under Album '{album_display}'" ) else: # Few tracks in album → treat as a “Singles” release - base_folder = os.path.join(artist_folder, f"{sanitize(primary)} - Singles") + base_folder = os.path.join(artist_folder, f"{sanitize(primary_display)} - Singles") decision_log.append( f" Count {count_now} ≥ {COMMON_ARTIST_THRESHOLD}, but " - f"album_count={c2} ≤ 3 → group under '{primary} - Singles'" + f"album_count={c2} ≤ 3 → group under '{primary_display} - Singles'" ) + if album: + album_display = album_display_names.get((p_lower, album.lower()), album) + base_folder = _apply_part_of_set(base_folder, album_display, part_of_set, decision_log) + # Build filename for all “common” tracks basename = os.path.basename(old_path) if "/" in raw_artist or is_repeated(raw_artist): @@ -750,8 +817,12 @@ def _compute(path: str) -> tuple[int | None, str | None]: # ─── 4.4) Build “leftover_tags” for HTML preview (optional) ─────────────── leftover = set(folders) leftover.discard(primary) + leftover.discard(primary_display) if album: leftover.discard(album) + leftover.discard(album_display_names.get((p_lower, album.lower()), album)) + if part_of_set: + leftover.discard(part_of_set) leftover.discard(year) if genre: leftover.add(genre) From 08e2762019595795335bc28635b587fe7a7e9ec6 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 28 Dec 2025 18:50:00 -0500 Subject: [PATCH 187/606] Revert "Normalize fallback artist parsing and apply part-of-set subfolder logic" --- .../indexer_engine/music_indexer_api.py | 77 ++++--------------- music_indexer_api.py | 77 ++++--------------- 2 files changed, 28 insertions(+), 126 deletions(-) diff --git a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py index 840c102..2426ec5 100644 --- a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py +++ b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py @@ -75,8 +75,15 @@ def progress_callback(_c, _t, _m, _p): # Normalize artist identically to Phase 4 raw = (tags.get("artist") or "").strip() + if not raw: - raw = _fallback_artist_from_filename(fname) + name_only = os.path.splitext(fname)[0] + if "_" in name_only: + raw = name_only.split("_", 1)[0] + elif " - " in name_only: + raw = name_only.split(" - ", 1)[0] + else: + raw = name_only primary, _ = extract_primary_and_collabs(raw) p_lower = primary.lower() @@ -89,14 +96,6 @@ def sanitize(name: str) -> str: invalid = r'<>:"/\\|?*' return "".join(c for c in (name or "Unknown") if c not in invalid).strip() or "Unknown" -def _fallback_artist_from_filename(filename: str) -> str: - name_only = os.path.splitext(os.path.basename(filename))[0] - if "_" in name_only: - return name_only.split("_", 1)[0] - if " - " in name_only: - return name_only.split(" - ", 1)[0] - return name_only - def collapse_repeats(name: str) -> str: """ If name is something like 'DROELOEDROELOE', collapse to 'DROELOE'. @@ -125,46 +124,16 @@ def is_repeated(name: str) -> bool: return True return False -def _normalize_part_of_set(raw_value: str | None) -> str | None: - if not raw_value: - return None - text = str(raw_value).strip() - if not text: - return None - first = text.split("/", 1)[0].strip() - if not first: - return None - if first.isdigit(): - return f"Part {int(first)}" - return first - - -def _apply_part_of_set( - base_folder: str, - album: str, - part_of_set: str | None, - decision_log: list[str], -) -> str: - if part_of_set and album and os.path.basename(base_folder) == sanitize(album): - part_folder = sanitize(part_of_set) - decision_log.append( - f" → Part of set '{part_of_set}' detected; placed under subfolder '{part_folder}'" - ) - return os.path.join(base_folder, part_folder) - return base_folder - - def get_tags(path: str): """ - Read basic tags using Mutagen (artist, title, album, year, track, genre, part_of_set). + Read basic tags using Mutagen (artist, title, album, year, track, genre). Return a dict with those fields (or None if missing). """ try: audio = MutagenFile(path, easy=True) if not audio or not audio.tags: return {"artist": None, "title": None, "album": None, - "year": None, "track": None, "genre": None, - "part_of_set": None} + "year": None, "track": None, "genre": None} tags = audio.tags artist = tags.get("artist", [None])[0] title = tags.get("title", [None])[0] @@ -179,21 +148,11 @@ def get_tags(path: str): except Exception: track = None genre = tags.get("genre", [None])[0] - raw_part_of_set = ( - tags.get("partofset", [None])[0] - or tags.get("part_of_set", [None])[0] - or tags.get("part of set", [None])[0] - or tags.get("discnumber", [None])[0] - or tags.get("disc", [None])[0] - ) - part_of_set = _normalize_part_of_set(raw_part_of_set) return {"artist": artist, "title": title, "album": album, - "year": year, "track": track, "genre": genre, - "part_of_set": part_of_set} + "year": year, "track": track, "genre": genre} except Exception: return {"artist": None, "title": None, "album": None, - "year": None, "track": None, "genre": None, - "part_of_set": None} + "year": None, "track": None, "genre": None} def extract_primary_and_collabs(raw_artist: str): """ @@ -420,7 +379,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: if idx % 50 == 0 or idx == total_audio: log_callback(f" • Processing file {idx}/{total_audio} for dedupe") data = get_tags(fullpath) - raw_artist = data["artist"] or _fallback_artist_from_filename(fullpath) + raw_artist = data["artist"] or os.path.splitext(os.path.basename(fullpath))[0] raw_artist = collapse_repeats(raw_artist) title = data["title"] or os.path.splitext(os.path.basename(fullpath))[0] album = data["album"] or "" @@ -553,14 +512,13 @@ def _compute(path: str) -> tuple[int | None, str | None]: progress_callback(idx, total_kept, f"Metadata {os.path.relpath(fullpath, root_path)}", "B") data = get_tags(fullpath) - raw_artist = data["artist"] or _fallback_artist_from_filename(fullpath) + raw_artist = data["artist"] or os.path.splitext(os.path.basename(fullpath))[0] raw_artist = collapse_repeats(raw_artist) title = data["title"] or os.path.splitext(os.path.basename(fullpath))[0] album = data["album"] year = data["year"] genre = data["genre"] track = data["track"] - part_of_set = data["part_of_set"] # 1) Primary & collabs (preserve original case) primary, collabs = extract_primary_and_collabs(raw_artist) @@ -608,7 +566,6 @@ def _compute(path: str) -> tuple[int | None, str | None]: "year": year, "genre": genre, "track": track, - "part_of_set": part_of_set, "cover_hash": cover_hash, "folder_tags": folder_tags, "missing_core": not data.get("artist") or not data.get("title"), @@ -637,7 +594,6 @@ def _compute(path: str) -> tuple[int | None, str | None]: album = info["album"] or "" year = info["year"] or "Unknown" track = info["track"] - part_of_set = info.get("part_of_set") folders = info["folder_tags"] if info.get("missing_core"): candidate = os.path.join(review_root, os.path.basename(old_path)) @@ -706,7 +662,6 @@ def _compute(path: str) -> tuple[int | None, str | None]: ) artist_folder = os.path.join(MUSIC_ROOT, "By Artist", sanitize(main_artist)) base_folder = os.path.join(artist_folder, sanitize(album)) - base_folder = _apply_part_of_set(base_folder, album, part_of_set, decision_log) # Build filename (skip normal album logic) basename = os.path.basename(old_path) @@ -773,8 +728,6 @@ def _compute(path: str) -> tuple[int | None, str | None]: f"album_count={c2} ≤ 3 → group under '{primary} - Singles'" ) - base_folder = _apply_part_of_set(base_folder, album, part_of_set, decision_log) - # Build filename for all “common” tracks basename = os.path.basename(old_path) if "/" in raw_artist or is_repeated(raw_artist): @@ -799,8 +752,6 @@ def _compute(path: str) -> tuple[int | None, str | None]: leftover.discard(primary) if album: leftover.discard(album) - if part_of_set: - leftover.discard(part_of_set) leftover.discard(year) if genre: leftover.add(genre) diff --git a/music_indexer_api.py b/music_indexer_api.py index 840c102..2426ec5 100644 --- a/music_indexer_api.py +++ b/music_indexer_api.py @@ -75,8 +75,15 @@ def progress_callback(_c, _t, _m, _p): # Normalize artist identically to Phase 4 raw = (tags.get("artist") or "").strip() + if not raw: - raw = _fallback_artist_from_filename(fname) + name_only = os.path.splitext(fname)[0] + if "_" in name_only: + raw = name_only.split("_", 1)[0] + elif " - " in name_only: + raw = name_only.split(" - ", 1)[0] + else: + raw = name_only primary, _ = extract_primary_and_collabs(raw) p_lower = primary.lower() @@ -89,14 +96,6 @@ def sanitize(name: str) -> str: invalid = r'<>:"/\\|?*' return "".join(c for c in (name or "Unknown") if c not in invalid).strip() or "Unknown" -def _fallback_artist_from_filename(filename: str) -> str: - name_only = os.path.splitext(os.path.basename(filename))[0] - if "_" in name_only: - return name_only.split("_", 1)[0] - if " - " in name_only: - return name_only.split(" - ", 1)[0] - return name_only - def collapse_repeats(name: str) -> str: """ If name is something like 'DROELOEDROELOE', collapse to 'DROELOE'. @@ -125,46 +124,16 @@ def is_repeated(name: str) -> bool: return True return False -def _normalize_part_of_set(raw_value: str | None) -> str | None: - if not raw_value: - return None - text = str(raw_value).strip() - if not text: - return None - first = text.split("/", 1)[0].strip() - if not first: - return None - if first.isdigit(): - return f"Part {int(first)}" - return first - - -def _apply_part_of_set( - base_folder: str, - album: str, - part_of_set: str | None, - decision_log: list[str], -) -> str: - if part_of_set and album and os.path.basename(base_folder) == sanitize(album): - part_folder = sanitize(part_of_set) - decision_log.append( - f" → Part of set '{part_of_set}' detected; placed under subfolder '{part_folder}'" - ) - return os.path.join(base_folder, part_folder) - return base_folder - - def get_tags(path: str): """ - Read basic tags using Mutagen (artist, title, album, year, track, genre, part_of_set). + Read basic tags using Mutagen (artist, title, album, year, track, genre). Return a dict with those fields (or None if missing). """ try: audio = MutagenFile(path, easy=True) if not audio or not audio.tags: return {"artist": None, "title": None, "album": None, - "year": None, "track": None, "genre": None, - "part_of_set": None} + "year": None, "track": None, "genre": None} tags = audio.tags artist = tags.get("artist", [None])[0] title = tags.get("title", [None])[0] @@ -179,21 +148,11 @@ def get_tags(path: str): except Exception: track = None genre = tags.get("genre", [None])[0] - raw_part_of_set = ( - tags.get("partofset", [None])[0] - or tags.get("part_of_set", [None])[0] - or tags.get("part of set", [None])[0] - or tags.get("discnumber", [None])[0] - or tags.get("disc", [None])[0] - ) - part_of_set = _normalize_part_of_set(raw_part_of_set) return {"artist": artist, "title": title, "album": album, - "year": year, "track": track, "genre": genre, - "part_of_set": part_of_set} + "year": year, "track": track, "genre": genre} except Exception: return {"artist": None, "title": None, "album": None, - "year": None, "track": None, "genre": None, - "part_of_set": None} + "year": None, "track": None, "genre": None} def extract_primary_and_collabs(raw_artist: str): """ @@ -420,7 +379,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: if idx % 50 == 0 or idx == total_audio: log_callback(f" • Processing file {idx}/{total_audio} for dedupe") data = get_tags(fullpath) - raw_artist = data["artist"] or _fallback_artist_from_filename(fullpath) + raw_artist = data["artist"] or os.path.splitext(os.path.basename(fullpath))[0] raw_artist = collapse_repeats(raw_artist) title = data["title"] or os.path.splitext(os.path.basename(fullpath))[0] album = data["album"] or "" @@ -553,14 +512,13 @@ def _compute(path: str) -> tuple[int | None, str | None]: progress_callback(idx, total_kept, f"Metadata {os.path.relpath(fullpath, root_path)}", "B") data = get_tags(fullpath) - raw_artist = data["artist"] or _fallback_artist_from_filename(fullpath) + raw_artist = data["artist"] or os.path.splitext(os.path.basename(fullpath))[0] raw_artist = collapse_repeats(raw_artist) title = data["title"] or os.path.splitext(os.path.basename(fullpath))[0] album = data["album"] year = data["year"] genre = data["genre"] track = data["track"] - part_of_set = data["part_of_set"] # 1) Primary & collabs (preserve original case) primary, collabs = extract_primary_and_collabs(raw_artist) @@ -608,7 +566,6 @@ def _compute(path: str) -> tuple[int | None, str | None]: "year": year, "genre": genre, "track": track, - "part_of_set": part_of_set, "cover_hash": cover_hash, "folder_tags": folder_tags, "missing_core": not data.get("artist") or not data.get("title"), @@ -637,7 +594,6 @@ def _compute(path: str) -> tuple[int | None, str | None]: album = info["album"] or "" year = info["year"] or "Unknown" track = info["track"] - part_of_set = info.get("part_of_set") folders = info["folder_tags"] if info.get("missing_core"): candidate = os.path.join(review_root, os.path.basename(old_path)) @@ -706,7 +662,6 @@ def _compute(path: str) -> tuple[int | None, str | None]: ) artist_folder = os.path.join(MUSIC_ROOT, "By Artist", sanitize(main_artist)) base_folder = os.path.join(artist_folder, sanitize(album)) - base_folder = _apply_part_of_set(base_folder, album, part_of_set, decision_log) # Build filename (skip normal album logic) basename = os.path.basename(old_path) @@ -773,8 +728,6 @@ def _compute(path: str) -> tuple[int | None, str | None]: f"album_count={c2} ≤ 3 → group under '{primary} - Singles'" ) - base_folder = _apply_part_of_set(base_folder, album, part_of_set, decision_log) - # Build filename for all “common” tracks basename = os.path.basename(old_path) if "/" in raw_artist or is_repeated(raw_artist): @@ -799,8 +752,6 @@ def _compute(path: str) -> tuple[int | None, str | None]: leftover.discard(primary) if album: leftover.discard(album) - if part_of_set: - leftover.discard(part_of_set) leftover.discard(year) if genre: leftover.add(genre) From 3315e894dcb8405bc82dd0f9125bc21453a8d4c4 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 28 Dec 2025 19:03:54 -0500 Subject: [PATCH 188/606] Add artwork thumbnails to execution report --- duplicate_consolidation_executor.py | 68 +++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index be560bb..444a68c 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -12,11 +12,13 @@ """ from __future__ import annotations +import base64 import datetime import hashlib import html import importlib.util import json +import mimetypes import platform import os import shutil @@ -274,6 +276,40 @@ def _extract_artwork_bytes(path: str) -> bytes | None: return None +def _artwork_data_uri_for_group(group: object | None) -> str | None: + if group is None: + return None + + candidate_paths: List[str] = [] + chosen_source = getattr(group, "chosen_artwork_source", None) + if isinstance(chosen_source, Mapping): + chosen_path = chosen_source.get("path") + if isinstance(chosen_path, str) and chosen_path: + candidate_paths.append(chosen_path) + + for art in getattr(group, "artwork", []) or []: + source = getattr(art, "source", None) + if not source and isinstance(art, Mapping): + source = art.get("source") + if isinstance(source, str) and source: + candidate_paths.append(source) + + for candidate in getattr(group, "artwork_candidates", []) or []: + path = getattr(candidate, "path", None) + if not path and isinstance(candidate, Mapping): + path = candidate.get("path") + if isinstance(path, str) and path: + candidate_paths.append(path) + + for path in candidate_paths: + payload = _extract_artwork_bytes(path) + if payload: + mime = mimetypes.guess_type(path)[0] or "image/jpeg" + encoded = base64.b64encode(payload).decode("ascii") + return f"data:{mime};base64,{encoded}" + return None + + def _apply_artwork(action: ArtworkDirective) -> tuple[bool, str]: payload = _extract_artwork_bytes(action.source) if payload is None: @@ -1250,8 +1286,33 @@ def _run_status() -> tuple[str, str, str]: "grid-template-columns: 1fr;", "gap:8px;", "background: #fff;", + "position: relative;", "}", "summary.group-summary::-webkit-details-marker{ display:none; }", + ".group-artwork{", + "position:absolute;", + "right:12px;", + "bottom:12px;", + "width:40px;", + "height:40px;", + "border-radius: 8px;", + "overflow:hidden;", + "pointer-events:none;", + "box-shadow: 0 0 0 1px rgba(0,0,0,0.04);", + "}", + ".group-artwork img{", + "width:100%;", + "height:100%;", + "object-fit:cover;", + "display:block;", + "}", + ".group-artwork::after{", + "content:'';", + "position:absolute;", + "inset:0;", + "border-radius: 8px;", + "box-shadow: inset 0 0 14px rgba(0,0,0,0.35);", + "}", ".group-top{", "display:flex;", "flex-wrap:wrap;", @@ -1452,6 +1513,7 @@ def _run_status() -> tuple[str, str, str]: if meta_notes: search_tokens.append(meta_notes) search_text = " ".join(str(token) for token in search_tokens if token) + artwork_uri = _artwork_data_uri_for_group(grp) summary_line = ( f"Status: {state}. " + "; ".join(badges) if badges else f"Status: {state}." ) @@ -1465,6 +1527,12 @@ def _run_status() -> tuple[str, str, str]: f"data-quarantine-count='{group_quarantine_count}'>" ) html_lines.append("") + if artwork_uri: + html_lines.append( + "
      " + f"" + "
      " + ) html_lines.append("
      ") html_lines.append("
      ") html_lines.append(f"Group {html.escape(gid)}") From 2df54dfbf8dfcd48ce9a9aad2c7ffffa55be44cd Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 28 Dec 2025 19:29:10 -0500 Subject: [PATCH 189/606] Auto-open duplicate preview and enable reports --- main_gui.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/main_gui.py b/main_gui.py index ffcd6bb..8660c24 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1919,11 +1919,15 @@ def _handle_scan(self) -> None: def _handle_preview(self) -> None: self._log_action("Preview clicked") self._generate_plan(write_preview=True) + if self.preview_json_path: + self._open_preview() def _handle_execute(self) -> None: path = self._validate_library_root() if not path: return + self.execution_report_path = None + self.open_report_btn.config(state="disabled") preview_plan_path = self.preview_json_path or os.path.join(path, "Docs", "duplicate_preview.json") plan_input = self._plan plan_for_checks = self._plan @@ -2015,7 +2019,9 @@ def finish(result, error: Exception | None = None) -> None: self._log_action(f"Execution complete: {'success' if result.success else 'failed'}") report_path = result.report_paths.get("html_report") self._log_action(f"Execution report: {report_path}") - if report_path and os.path.exists(ensure_long_path(report_path)): + if report_path: + if not os.path.exists(ensure_long_path(report_path)): + self._log_action("Execution report missing at expected path; enabling Open Report anyway.") self.execution_report_path = report_path self.open_report_btn.config(state="normal") else: @@ -2092,6 +2098,14 @@ def _reset_preview_if_needed(self) -> None: self.preview_json_path = None self._log_action("Preview cleared; generate a new preview to reflect delete settings.") + def _open_preview(self) -> None: + self._open_local_html( + self.preview_json_path, + title="Preview Output", + missing_message="Preview output could not be found. Generate a new preview.", + empty_message="No preview output is available yet.", + ) + def _open_execution_report(self) -> None: self._open_local_html( self.execution_report_path, From 30a431834e9ab58b3ca0ec4c4fc636e7c13ecc14 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 28 Dec 2025 19:58:57 -0500 Subject: [PATCH 190/606] Add HTML output for duplicate preview --- duplicate_consolidation.py | 108 +++++++++++++++++++++++++++++++++++++ main_gui.py | 18 ++++++- 2 files changed, 124 insertions(+), 2 deletions(-) diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index 01997b2..a8ea53d 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -9,6 +9,7 @@ from __future__ import annotations import datetime +import html import hashlib import importlib.util import io @@ -1765,3 +1766,110 @@ def export_consolidation_preview(plan: ConsolidationPlan, output_json_path: str) with open(output_json_path, "w", encoding="utf-8") as f: json.dump(payload, f, indent=2) return output_json_path + + +def export_consolidation_preview_html(plan: ConsolidationPlan, output_html_path: str) -> str: + """Write an HTML preview of the consolidation plan.""" + + def esc(value: object) -> str: + return html.escape(str(value)) + + summary = { + "Groups": len(plan.groups), + "Review required": plan.review_required_count, + "Placeholders present": "Yes" if plan.placeholders_present else "No", + "Generated at": plan.generated_at.isoformat(), + } + + html_lines = [ + "", + "", + "", + "", + "", + "Duplicate Finder Preview", + "", + "", + "", + "

      Duplicate Finder Preview

      ", + "

      Review the duplicate consolidation plan before executing.

      ", + "
      ", + ] + for label, value in summary.items(): + html_lines.append("
      ") + html_lines.append(f"
      {esc(label)}
      ") + html_lines.append(f"
      {esc(value)}
      ") + html_lines.append("
      ") + html_lines.append("
      ") + + if plan.review_flags: + html_lines.append("
      ") + html_lines.append("

      Global review flags

      ") + html_lines.append("
        ") + for flag in plan.review_flags: + html_lines.append(f"
      • {esc(flag)}
      • ") + html_lines.append("
      ") + html_lines.append("
      ") + + html_lines.extend( + [ + "
      ", + "

      Duplicate groups

      ", + "", + "", + "", + ] + ) + for group in plan.groups: + status = "Review" if group.review_flags else "Ready" + badge_class = "review" if group.review_flags else "ready" + html_lines.append("") + html_lines.append(f"") + html_lines.append(f"") + html_lines.append(f"") + html_lines.append(f"") + html_lines.append("") + html_lines.append("
      GroupWinnerLosersStatus
      {esc(group.group_id)}{esc(os.path.basename(group.winner_path))}{esc(len(group.losers))}{esc(status)}
      ") + + for group in plan.groups: + html_lines.append("
      ") + html_lines.append(f"Group {esc(group.group_id)}") + html_lines.append("
      ") + html_lines.append(f"Winner:\n{esc(group.winner_path)}\n") + if group.losers: + html_lines.append("Losers:") + for loser in group.losers: + html_lines.append(f"\n- {esc(loser)}") + if group.review_flags: + html_lines.append("\n\nReview flags:") + for flag in group.review_flags: + html_lines.append(f"\n- {esc(flag)}") + html_lines.append("
      ") + html_lines.append("
      ") + + html_lines.extend(["
      ", "", ""]) + + with open(ensure_long_path(output_html_path), "w", encoding="utf-8") as handle: + handle.write("\n".join(html_lines)) + return output_html_path diff --git a/main_gui.py b/main_gui.py index 8660c24..7049167 100644 --- a/main_gui.py +++ b/main_gui.py @@ -46,6 +46,7 @@ build_consolidation_plan, consolidation_plan_from_dict, export_consolidation_preview, + export_consolidation_preview_html, ) from duplicate_consolidation_executor import ExecutionConfig, execute_consolidation_plan from controllers.library_index_controller import generate_index @@ -1476,6 +1477,7 @@ def __init__(self, parent: tk.Widget, library_path: str): self.group_disposition_overrides: dict[str, str] = {} self._selected_group_id: str | None = None self.preview_json_path: str | None = None + self.preview_html_path: str | None = None self.execution_report_path: str | None = None self._plan = None @@ -1882,9 +1884,14 @@ def _generate_plan(self, write_preview: bool) -> None: export_consolidation_preview(plan, json_path) self.preview_json_path = json_path self._log_action(f"Audit JSON written to {json_path}") + html_path = os.path.join(docs_dir, "duplicate_preview.html") + export_consolidation_preview_html(plan, html_path) + self.preview_html_path = html_path + self._log_action(f"Preview HTML written to {html_path}") self._set_status("Preview generated", progress=100) else: self.preview_json_path = None + self.preview_html_path = None self._set_status("Plan ready", progress=100) self._log_action( @@ -1919,7 +1926,7 @@ def _handle_scan(self) -> None: def _handle_preview(self) -> None: self._log_action("Preview clicked") self._generate_plan(write_preview=True) - if self.preview_json_path: + if self.preview_html_path: self._open_preview() def _handle_execute(self) -> None: @@ -1934,6 +1941,12 @@ def _handle_execute(self) -> None: if preview_plan_path and os.path.exists(preview_plan_path): if not self.preview_json_path: self.preview_json_path = preview_plan_path + if not self.preview_html_path: + preview_html_path = os.path.join(path, "Docs", "duplicate_preview.html") + plan_for_html = plan_for_checks or self._load_preview_plan(preview_plan_path) + if plan_for_html: + export_consolidation_preview_html(plan_for_html, preview_html_path) + self.preview_html_path = preview_html_path if not plan_for_checks: plan_for_checks = self._load_preview_plan(preview_plan_path) if plan_for_checks: @@ -2096,11 +2109,12 @@ def _apply_deletion_mode(self) -> None: def _reset_preview_if_needed(self) -> None: if self.preview_json_path: self.preview_json_path = None + self.preview_html_path = None self._log_action("Preview cleared; generate a new preview to reflect delete settings.") def _open_preview(self) -> None: self._open_local_html( - self.preview_json_path, + self.preview_html_path, title="Preview Output", missing_message="Preview output could not be found. Generate a new preview.", empty_message="No preview output is available yet.", From e485d51a06851e3be28cc6623e5f65f204d153cf Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 28 Dec 2025 20:01:28 -0500 Subject: [PATCH 191/606] Revert "Normalize artist/album display names, unify filename fallback, and apply part_of_set logic" --- .../indexer_engine/music_indexer_api.py | 123 ++++-------------- music_indexer_api.py | 123 ++++-------------- 2 files changed, 52 insertions(+), 194 deletions(-) diff --git a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py index 3c79d81..2426ec5 100644 --- a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py +++ b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py @@ -75,8 +75,15 @@ def progress_callback(_c, _t, _m, _p): # Normalize artist identically to Phase 4 raw = (tags.get("artist") or "").strip() + if not raw: - raw = _fallback_artist_from_filename(fname) + name_only = os.path.splitext(fname)[0] + if "_" in name_only: + raw = name_only.split("_", 1)[0] + elif " - " in name_only: + raw = name_only.split(" - ", 1)[0] + else: + raw = name_only primary, _ = extract_primary_and_collabs(raw) p_lower = primary.lower() @@ -89,14 +96,6 @@ def sanitize(name: str) -> str: invalid = r'<>:"/\\|?*' return "".join(c for c in (name or "Unknown") if c not in invalid).strip() or "Unknown" -def _fallback_artist_from_filename(filename: str) -> str: - name_only = os.path.splitext(os.path.basename(filename))[0] - if "_" in name_only: - return name_only.split("_", 1)[0] - if " - " in name_only: - return name_only.split(" - ", 1)[0] - return name_only - def collapse_repeats(name: str) -> str: """ If name is something like 'DROELOEDROELOE', collapse to 'DROELOE'. @@ -125,46 +124,16 @@ def is_repeated(name: str) -> bool: return True return False -def _normalize_part_of_set(raw_value: str | None) -> str | None: - if not raw_value: - return None - text = str(raw_value).strip() - if not text: - return None - first = text.split("/", 1)[0].strip() - if not first: - return None - if first.isdigit(): - return f"Part {int(first)}" - return first - - -def _apply_part_of_set( - base_folder: str, - album_folder: str, - part_of_set: str | None, - decision_log: list[str], -) -> str: - if part_of_set and album_folder and os.path.basename(base_folder) == sanitize(album_folder): - part_folder = sanitize(part_of_set) - decision_log.append( - f" → Part of set '{part_of_set}' detected; placed under subfolder '{part_folder}'" - ) - return os.path.join(base_folder, part_folder) - return base_folder - - def get_tags(path: str): """ - Read basic tags using Mutagen (artist, title, album, year, track, genre, part_of_set). + Read basic tags using Mutagen (artist, title, album, year, track, genre). Return a dict with those fields (or None if missing). """ try: audio = MutagenFile(path, easy=True) if not audio or not audio.tags: return {"artist": None, "title": None, "album": None, - "year": None, "track": None, "genre": None, - "part_of_set": None} + "year": None, "track": None, "genre": None} tags = audio.tags artist = tags.get("artist", [None])[0] title = tags.get("title", [None])[0] @@ -179,21 +148,11 @@ def get_tags(path: str): except Exception: track = None genre = tags.get("genre", [None])[0] - raw_part_of_set = ( - tags.get("partofset", [None])[0] - or tags.get("part_of_set", [None])[0] - or tags.get("part of set", [None])[0] - or tags.get("discnumber", [None])[0] - or tags.get("disc", [None])[0] - ) - part_of_set = _normalize_part_of_set(raw_part_of_set) return {"artist": artist, "title": title, "album": album, - "year": year, "track": track, "genre": genre, - "part_of_set": part_of_set} + "year": year, "track": track, "genre": genre} except Exception: return {"artist": None, "title": None, "album": None, - "year": None, "track": None, "genre": None, - "part_of_set": None} + "year": None, "track": None, "genre": None} def extract_primary_and_collabs(raw_artist: str): """ @@ -420,7 +379,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: if idx % 50 == 0 or idx == total_audio: log_callback(f" • Processing file {idx}/{total_audio} for dedupe") data = get_tags(fullpath) - raw_artist = data["artist"] or _fallback_artist_from_filename(fullpath) + raw_artist = data["artist"] or os.path.splitext(os.path.basename(fullpath))[0] raw_artist = collapse_repeats(raw_artist) title = data["title"] or os.path.splitext(os.path.basename(fullpath))[0] album = data["album"] or "" @@ -544,8 +503,6 @@ def _compute(path: str) -> tuple[int | None, str | None]: album_counts = defaultdict(int) remix_counts = defaultdict(int) cover_counts = defaultdict(int) - primary_display_counts: Dict[str, Dict[str, int]] = defaultdict(lambda: defaultdict(int)) - album_display_counts: Dict[tuple[str, str], Dict[str, int]] = defaultdict(lambda: defaultdict(int)) total_kept = len(kept_files) progress_callback(0, total_kept, "Reading metadata", "B") @@ -555,23 +512,18 @@ def _compute(path: str) -> tuple[int | None, str | None]: progress_callback(idx, total_kept, f"Metadata {os.path.relpath(fullpath, root_path)}", "B") data = get_tags(fullpath) - raw_artist = data["artist"] or _fallback_artist_from_filename(fullpath) + raw_artist = data["artist"] or os.path.splitext(os.path.basename(fullpath))[0] raw_artist = collapse_repeats(raw_artist) title = data["title"] or os.path.splitext(os.path.basename(fullpath))[0] album = data["album"] year = data["year"] genre = data["genre"] track = data["track"] - part_of_set = data["part_of_set"] # 1) Primary & collabs (preserve original case) primary, collabs = extract_primary_and_collabs(raw_artist) p_lower = primary.lower() - primary_display_counts[p_lower][primary] += 1 - if album: - album_display_counts[(p_lower, album.lower())][album] += 1 - # 2) album_counts for genuine (non-remix) tracks under each (artist, album) if album and "remix" not in title.lower() and album.strip().lower() != title.strip().lower(): album_counts[(p_lower, album.lower())] += 1 @@ -614,21 +566,12 @@ def _compute(path: str) -> tuple[int | None, str | None]: "year": year, "genre": genre, "track": track, - "part_of_set": part_of_set, "cover_hash": cover_hash, "folder_tags": folder_tags, "missing_core": not data.get("artist") or not data.get("title"), } log_callback(f" → Collected metadata for {total_kept} files.") - primary_display_names = { - key: max(values, key=lambda name: (values[name], len(name))) - for key, values in primary_display_counts.items() - } - album_display_names = { - key: max(values, key=lambda name: (values[name], len(name))) - for key, values in album_display_counts.items() - } # ─── Phase 4: Determine destination for each file (with logging) ───────────── log_callback("4/6: Determining destination paths for each file…") @@ -651,7 +594,6 @@ def _compute(path: str) -> tuple[int | None, str | None]: album = info["album"] or "" year = info["year"] or "Unknown" track = info["track"] - part_of_set = info.get("part_of_set") folders = info["folder_tags"] if info.get("missing_core"): candidate = os.path.join(review_root, os.path.basename(old_path)) @@ -714,15 +656,12 @@ def _compute(path: str) -> tuple[int | None, str | None]: # Promote primary to remixer (first part of raw_artist before “/”) main_artist = raw_artist.split("/", 1)[0] p_lower = main_artist.lower() - main_artist_display = primary_display_names.get(p_lower, main_artist) decision_log.append( f" → Enough remixes ({rcount} ≥ {REMIX_FOLDER_THRESHOLD}) " - f"AND count_now ({count_now}) ≥ {COMMON_ARTIST_THRESHOLD}; force primary = '{main_artist_display}'" + f"AND count_now ({count_now}) ≥ {COMMON_ARTIST_THRESHOLD}; force primary = '{main_artist}'" ) - artist_folder = os.path.join(MUSIC_ROOT, "By Artist", sanitize(main_artist_display)) - album_display = album_display_names.get((p_lower, album.lower()), album) - base_folder = os.path.join(artist_folder, sanitize(album_display)) - base_folder = _apply_part_of_set(base_folder, album_display, part_of_set, decision_log) + artist_folder = os.path.join(MUSIC_ROOT, "By Artist", sanitize(main_artist)) + base_folder = os.path.join(artist_folder, sanitize(album)) # Build filename (skip normal album logic) basename = os.path.basename(old_path) @@ -740,7 +679,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: new_path = os.path.join(base_folder, new_filename) moves[old_path] = new_path decision_log.append( - f" → (Remixes folder) placed under By Artist/{main_artist_display}/{album_display} " + f" → (Remixes folder) placed under By Artist/{main_artist}/{album} " f"→ Final: '{os.path.relpath(new_path, MUSIC_ROOT)}'" ) continue @@ -758,43 +697,37 @@ def _compute(path: str) -> tuple[int | None, str | None]: decision_log.append(f" Candidates: {all_candidates}, counts: {counts}") primary = best_artist p_lower = primary.lower() - primary_display = primary_display_names.get(p_lower, primary) - decision_log.append(f" → After ranking, primary = '{primary_display}'") + decision_log.append(f" → After ranking, primary = '{primary}'") # Build “By Artist/” base path - artist_folder = os.path.join(MUSIC_ROOT, "By Artist", sanitize(primary_display)) + artist_folder = os.path.join(MUSIC_ROOT, "By Artist", sanitize(primary)) # ─── 4.4) COMMON ARTIST: decide between “Album” vs. “Singles” ─────────────── # (At this point, count_now ≥ COMMON_ARTIST_THRESHOLD is guaranteed.) if not album or album.strip().lower() == title.strip().lower(): # No album tag (or album == title) → put into “ - Singles” - base_folder = os.path.join(artist_folder, f"{sanitize(primary_display)} - Singles") + base_folder = os.path.join(artist_folder, f"{sanitize(primary)} - Singles") decision_log.append( f" Count {count_now} ≥ {COMMON_ARTIST_THRESHOLD}, no album or album=title " - f"→ group under '{primary_display} - Singles'" + f"→ group under '{primary} - Singles'" ) else: c2 = album_counts.get((p_lower, album.lower()), 0) if c2 > 3: # Enough tracks in this album → put into “/” - album_display = album_display_names.get((p_lower, album.lower()), album) - base_folder = os.path.join(artist_folder, sanitize(album_display)) + base_folder = os.path.join(artist_folder, sanitize(album)) decision_log.append( f" Count {count_now} ≥ {COMMON_ARTIST_THRESHOLD}, album_count={c2} > 3 " - f"→ group under Album '{album_display}'" + f"→ group under Album '{album}'" ) else: # Few tracks in album → treat as a “Singles” release - base_folder = os.path.join(artist_folder, f"{sanitize(primary_display)} - Singles") + base_folder = os.path.join(artist_folder, f"{sanitize(primary)} - Singles") decision_log.append( f" Count {count_now} ≥ {COMMON_ARTIST_THRESHOLD}, but " - f"album_count={c2} ≤ 3 → group under '{primary_display} - Singles'" + f"album_count={c2} ≤ 3 → group under '{primary} - Singles'" ) - if album: - album_display = album_display_names.get((p_lower, album.lower()), album) - base_folder = _apply_part_of_set(base_folder, album_display, part_of_set, decision_log) - # Build filename for all “common” tracks basename = os.path.basename(old_path) if "/" in raw_artist or is_repeated(raw_artist): @@ -817,12 +750,8 @@ def _compute(path: str) -> tuple[int | None, str | None]: # ─── 4.4) Build “leftover_tags” for HTML preview (optional) ─────────────── leftover = set(folders) leftover.discard(primary) - leftover.discard(primary_display) if album: leftover.discard(album) - leftover.discard(album_display_names.get((p_lower, album.lower()), album)) - if part_of_set: - leftover.discard(part_of_set) leftover.discard(year) if genre: leftover.add(genre) diff --git a/music_indexer_api.py b/music_indexer_api.py index 3c79d81..2426ec5 100644 --- a/music_indexer_api.py +++ b/music_indexer_api.py @@ -75,8 +75,15 @@ def progress_callback(_c, _t, _m, _p): # Normalize artist identically to Phase 4 raw = (tags.get("artist") or "").strip() + if not raw: - raw = _fallback_artist_from_filename(fname) + name_only = os.path.splitext(fname)[0] + if "_" in name_only: + raw = name_only.split("_", 1)[0] + elif " - " in name_only: + raw = name_only.split(" - ", 1)[0] + else: + raw = name_only primary, _ = extract_primary_and_collabs(raw) p_lower = primary.lower() @@ -89,14 +96,6 @@ def sanitize(name: str) -> str: invalid = r'<>:"/\\|?*' return "".join(c for c in (name or "Unknown") if c not in invalid).strip() or "Unknown" -def _fallback_artist_from_filename(filename: str) -> str: - name_only = os.path.splitext(os.path.basename(filename))[0] - if "_" in name_only: - return name_only.split("_", 1)[0] - if " - " in name_only: - return name_only.split(" - ", 1)[0] - return name_only - def collapse_repeats(name: str) -> str: """ If name is something like 'DROELOEDROELOE', collapse to 'DROELOE'. @@ -125,46 +124,16 @@ def is_repeated(name: str) -> bool: return True return False -def _normalize_part_of_set(raw_value: str | None) -> str | None: - if not raw_value: - return None - text = str(raw_value).strip() - if not text: - return None - first = text.split("/", 1)[0].strip() - if not first: - return None - if first.isdigit(): - return f"Part {int(first)}" - return first - - -def _apply_part_of_set( - base_folder: str, - album_folder: str, - part_of_set: str | None, - decision_log: list[str], -) -> str: - if part_of_set and album_folder and os.path.basename(base_folder) == sanitize(album_folder): - part_folder = sanitize(part_of_set) - decision_log.append( - f" → Part of set '{part_of_set}' detected; placed under subfolder '{part_folder}'" - ) - return os.path.join(base_folder, part_folder) - return base_folder - - def get_tags(path: str): """ - Read basic tags using Mutagen (artist, title, album, year, track, genre, part_of_set). + Read basic tags using Mutagen (artist, title, album, year, track, genre). Return a dict with those fields (or None if missing). """ try: audio = MutagenFile(path, easy=True) if not audio or not audio.tags: return {"artist": None, "title": None, "album": None, - "year": None, "track": None, "genre": None, - "part_of_set": None} + "year": None, "track": None, "genre": None} tags = audio.tags artist = tags.get("artist", [None])[0] title = tags.get("title", [None])[0] @@ -179,21 +148,11 @@ def get_tags(path: str): except Exception: track = None genre = tags.get("genre", [None])[0] - raw_part_of_set = ( - tags.get("partofset", [None])[0] - or tags.get("part_of_set", [None])[0] - or tags.get("part of set", [None])[0] - or tags.get("discnumber", [None])[0] - or tags.get("disc", [None])[0] - ) - part_of_set = _normalize_part_of_set(raw_part_of_set) return {"artist": artist, "title": title, "album": album, - "year": year, "track": track, "genre": genre, - "part_of_set": part_of_set} + "year": year, "track": track, "genre": genre} except Exception: return {"artist": None, "title": None, "album": None, - "year": None, "track": None, "genre": None, - "part_of_set": None} + "year": None, "track": None, "genre": None} def extract_primary_and_collabs(raw_artist: str): """ @@ -420,7 +379,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: if idx % 50 == 0 or idx == total_audio: log_callback(f" • Processing file {idx}/{total_audio} for dedupe") data = get_tags(fullpath) - raw_artist = data["artist"] or _fallback_artist_from_filename(fullpath) + raw_artist = data["artist"] or os.path.splitext(os.path.basename(fullpath))[0] raw_artist = collapse_repeats(raw_artist) title = data["title"] or os.path.splitext(os.path.basename(fullpath))[0] album = data["album"] or "" @@ -544,8 +503,6 @@ def _compute(path: str) -> tuple[int | None, str | None]: album_counts = defaultdict(int) remix_counts = defaultdict(int) cover_counts = defaultdict(int) - primary_display_counts: Dict[str, Dict[str, int]] = defaultdict(lambda: defaultdict(int)) - album_display_counts: Dict[tuple[str, str], Dict[str, int]] = defaultdict(lambda: defaultdict(int)) total_kept = len(kept_files) progress_callback(0, total_kept, "Reading metadata", "B") @@ -555,23 +512,18 @@ def _compute(path: str) -> tuple[int | None, str | None]: progress_callback(idx, total_kept, f"Metadata {os.path.relpath(fullpath, root_path)}", "B") data = get_tags(fullpath) - raw_artist = data["artist"] or _fallback_artist_from_filename(fullpath) + raw_artist = data["artist"] or os.path.splitext(os.path.basename(fullpath))[0] raw_artist = collapse_repeats(raw_artist) title = data["title"] or os.path.splitext(os.path.basename(fullpath))[0] album = data["album"] year = data["year"] genre = data["genre"] track = data["track"] - part_of_set = data["part_of_set"] # 1) Primary & collabs (preserve original case) primary, collabs = extract_primary_and_collabs(raw_artist) p_lower = primary.lower() - primary_display_counts[p_lower][primary] += 1 - if album: - album_display_counts[(p_lower, album.lower())][album] += 1 - # 2) album_counts for genuine (non-remix) tracks under each (artist, album) if album and "remix" not in title.lower() and album.strip().lower() != title.strip().lower(): album_counts[(p_lower, album.lower())] += 1 @@ -614,21 +566,12 @@ def _compute(path: str) -> tuple[int | None, str | None]: "year": year, "genre": genre, "track": track, - "part_of_set": part_of_set, "cover_hash": cover_hash, "folder_tags": folder_tags, "missing_core": not data.get("artist") or not data.get("title"), } log_callback(f" → Collected metadata for {total_kept} files.") - primary_display_names = { - key: max(values, key=lambda name: (values[name], len(name))) - for key, values in primary_display_counts.items() - } - album_display_names = { - key: max(values, key=lambda name: (values[name], len(name))) - for key, values in album_display_counts.items() - } # ─── Phase 4: Determine destination for each file (with logging) ───────────── log_callback("4/6: Determining destination paths for each file…") @@ -651,7 +594,6 @@ def _compute(path: str) -> tuple[int | None, str | None]: album = info["album"] or "" year = info["year"] or "Unknown" track = info["track"] - part_of_set = info.get("part_of_set") folders = info["folder_tags"] if info.get("missing_core"): candidate = os.path.join(review_root, os.path.basename(old_path)) @@ -714,15 +656,12 @@ def _compute(path: str) -> tuple[int | None, str | None]: # Promote primary to remixer (first part of raw_artist before “/”) main_artist = raw_artist.split("/", 1)[0] p_lower = main_artist.lower() - main_artist_display = primary_display_names.get(p_lower, main_artist) decision_log.append( f" → Enough remixes ({rcount} ≥ {REMIX_FOLDER_THRESHOLD}) " - f"AND count_now ({count_now}) ≥ {COMMON_ARTIST_THRESHOLD}; force primary = '{main_artist_display}'" + f"AND count_now ({count_now}) ≥ {COMMON_ARTIST_THRESHOLD}; force primary = '{main_artist}'" ) - artist_folder = os.path.join(MUSIC_ROOT, "By Artist", sanitize(main_artist_display)) - album_display = album_display_names.get((p_lower, album.lower()), album) - base_folder = os.path.join(artist_folder, sanitize(album_display)) - base_folder = _apply_part_of_set(base_folder, album_display, part_of_set, decision_log) + artist_folder = os.path.join(MUSIC_ROOT, "By Artist", sanitize(main_artist)) + base_folder = os.path.join(artist_folder, sanitize(album)) # Build filename (skip normal album logic) basename = os.path.basename(old_path) @@ -740,7 +679,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: new_path = os.path.join(base_folder, new_filename) moves[old_path] = new_path decision_log.append( - f" → (Remixes folder) placed under By Artist/{main_artist_display}/{album_display} " + f" → (Remixes folder) placed under By Artist/{main_artist}/{album} " f"→ Final: '{os.path.relpath(new_path, MUSIC_ROOT)}'" ) continue @@ -758,43 +697,37 @@ def _compute(path: str) -> tuple[int | None, str | None]: decision_log.append(f" Candidates: {all_candidates}, counts: {counts}") primary = best_artist p_lower = primary.lower() - primary_display = primary_display_names.get(p_lower, primary) - decision_log.append(f" → After ranking, primary = '{primary_display}'") + decision_log.append(f" → After ranking, primary = '{primary}'") # Build “By Artist/” base path - artist_folder = os.path.join(MUSIC_ROOT, "By Artist", sanitize(primary_display)) + artist_folder = os.path.join(MUSIC_ROOT, "By Artist", sanitize(primary)) # ─── 4.4) COMMON ARTIST: decide between “Album” vs. “Singles” ─────────────── # (At this point, count_now ≥ COMMON_ARTIST_THRESHOLD is guaranteed.) if not album or album.strip().lower() == title.strip().lower(): # No album tag (or album == title) → put into “ - Singles” - base_folder = os.path.join(artist_folder, f"{sanitize(primary_display)} - Singles") + base_folder = os.path.join(artist_folder, f"{sanitize(primary)} - Singles") decision_log.append( f" Count {count_now} ≥ {COMMON_ARTIST_THRESHOLD}, no album or album=title " - f"→ group under '{primary_display} - Singles'" + f"→ group under '{primary} - Singles'" ) else: c2 = album_counts.get((p_lower, album.lower()), 0) if c2 > 3: # Enough tracks in this album → put into “/” - album_display = album_display_names.get((p_lower, album.lower()), album) - base_folder = os.path.join(artist_folder, sanitize(album_display)) + base_folder = os.path.join(artist_folder, sanitize(album)) decision_log.append( f" Count {count_now} ≥ {COMMON_ARTIST_THRESHOLD}, album_count={c2} > 3 " - f"→ group under Album '{album_display}'" + f"→ group under Album '{album}'" ) else: # Few tracks in album → treat as a “Singles” release - base_folder = os.path.join(artist_folder, f"{sanitize(primary_display)} - Singles") + base_folder = os.path.join(artist_folder, f"{sanitize(primary)} - Singles") decision_log.append( f" Count {count_now} ≥ {COMMON_ARTIST_THRESHOLD}, but " - f"album_count={c2} ≤ 3 → group under '{primary_display} - Singles'" + f"album_count={c2} ≤ 3 → group under '{primary} - Singles'" ) - if album: - album_display = album_display_names.get((p_lower, album.lower()), album) - base_folder = _apply_part_of_set(base_folder, album_display, part_of_set, decision_log) - # Build filename for all “common” tracks basename = os.path.basename(old_path) if "/" in raw_artist or is_repeated(raw_artist): @@ -817,12 +750,8 @@ def _compute(path: str) -> tuple[int | None, str | None]: # ─── 4.4) Build “leftover_tags” for HTML preview (optional) ─────────────── leftover = set(folders) leftover.discard(primary) - leftover.discard(primary_display) if album: leftover.discard(album) - leftover.discard(album_display_names.get((p_lower, album.lower()), album)) - if part_of_set: - leftover.discard(part_of_set) leftover.discard(year) if genre: leftover.add(genre) From a4a7b01104932fe02c945c9fde2065f3b4bd2110 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 28 Dec 2025 20:01:52 -0500 Subject: [PATCH 192/606] Revert "Include album artwork thumbnails in execution report" --- duplicate_consolidation_executor.py | 68 ----------------------------- 1 file changed, 68 deletions(-) diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index 444a68c..be560bb 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -12,13 +12,11 @@ """ from __future__ import annotations -import base64 import datetime import hashlib import html import importlib.util import json -import mimetypes import platform import os import shutil @@ -276,40 +274,6 @@ def _extract_artwork_bytes(path: str) -> bytes | None: return None -def _artwork_data_uri_for_group(group: object | None) -> str | None: - if group is None: - return None - - candidate_paths: List[str] = [] - chosen_source = getattr(group, "chosen_artwork_source", None) - if isinstance(chosen_source, Mapping): - chosen_path = chosen_source.get("path") - if isinstance(chosen_path, str) and chosen_path: - candidate_paths.append(chosen_path) - - for art in getattr(group, "artwork", []) or []: - source = getattr(art, "source", None) - if not source and isinstance(art, Mapping): - source = art.get("source") - if isinstance(source, str) and source: - candidate_paths.append(source) - - for candidate in getattr(group, "artwork_candidates", []) or []: - path = getattr(candidate, "path", None) - if not path and isinstance(candidate, Mapping): - path = candidate.get("path") - if isinstance(path, str) and path: - candidate_paths.append(path) - - for path in candidate_paths: - payload = _extract_artwork_bytes(path) - if payload: - mime = mimetypes.guess_type(path)[0] or "image/jpeg" - encoded = base64.b64encode(payload).decode("ascii") - return f"data:{mime};base64,{encoded}" - return None - - def _apply_artwork(action: ArtworkDirective) -> tuple[bool, str]: payload = _extract_artwork_bytes(action.source) if payload is None: @@ -1286,33 +1250,8 @@ def _run_status() -> tuple[str, str, str]: "grid-template-columns: 1fr;", "gap:8px;", "background: #fff;", - "position: relative;", "}", "summary.group-summary::-webkit-details-marker{ display:none; }", - ".group-artwork{", - "position:absolute;", - "right:12px;", - "bottom:12px;", - "width:40px;", - "height:40px;", - "border-radius: 8px;", - "overflow:hidden;", - "pointer-events:none;", - "box-shadow: 0 0 0 1px rgba(0,0,0,0.04);", - "}", - ".group-artwork img{", - "width:100%;", - "height:100%;", - "object-fit:cover;", - "display:block;", - "}", - ".group-artwork::after{", - "content:'';", - "position:absolute;", - "inset:0;", - "border-radius: 8px;", - "box-shadow: inset 0 0 14px rgba(0,0,0,0.35);", - "}", ".group-top{", "display:flex;", "flex-wrap:wrap;", @@ -1513,7 +1452,6 @@ def _run_status() -> tuple[str, str, str]: if meta_notes: search_tokens.append(meta_notes) search_text = " ".join(str(token) for token in search_tokens if token) - artwork_uri = _artwork_data_uri_for_group(grp) summary_line = ( f"Status: {state}. " + "; ".join(badges) if badges else f"Status: {state}." ) @@ -1527,12 +1465,6 @@ def _run_status() -> tuple[str, str, str]: f"data-quarantine-count='{group_quarantine_count}'>" ) html_lines.append("") - if artwork_uri: - html_lines.append( - "
      " - f"" - "
      " - ) html_lines.append("
      ") html_lines.append("
      ") html_lines.append(f"Group {html.escape(gid)}") From 3a1452535017dc7456336513920b69c610ef0544 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 28 Dec 2025 20:09:50 -0500 Subject: [PATCH 193/606] Revert "Revert "Add part-of-set album subfolder support"" --- .../indexer_engine/music_indexer_api.py | 56 +++++++++++++++++-- music_indexer_api.py | 56 +++++++++++++++++-- 2 files changed, 104 insertions(+), 8 deletions(-) diff --git a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py index 2426ec5..449e611 100644 --- a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py +++ b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py @@ -124,16 +124,46 @@ def is_repeated(name: str) -> bool: return True return False +def _normalize_part_of_set(raw_value: str | None) -> str | None: + if not raw_value: + return None + text = str(raw_value).strip() + if not text: + return None + first = text.split("/", 1)[0].strip() + if not first: + return None + if first.isdigit(): + return f"Part {int(first)}" + return first + + +def _apply_part_of_set( + base_folder: str, + album: str, + part_of_set: str | None, + decision_log: list[str], +) -> str: + if part_of_set and album and os.path.basename(base_folder) == sanitize(album): + part_folder = sanitize(part_of_set) + decision_log.append( + f" → Part of set '{part_of_set}' detected; placed under subfolder '{part_folder}'" + ) + return os.path.join(base_folder, part_folder) + return base_folder + + def get_tags(path: str): """ - Read basic tags using Mutagen (artist, title, album, year, track, genre). + Read basic tags using Mutagen (artist, title, album, year, track, genre, part_of_set). Return a dict with those fields (or None if missing). """ try: audio = MutagenFile(path, easy=True) if not audio or not audio.tags: return {"artist": None, "title": None, "album": None, - "year": None, "track": None, "genre": None} + "year": None, "track": None, "genre": None, + "part_of_set": None} tags = audio.tags artist = tags.get("artist", [None])[0] title = tags.get("title", [None])[0] @@ -148,11 +178,21 @@ def get_tags(path: str): except Exception: track = None genre = tags.get("genre", [None])[0] + raw_part_of_set = ( + tags.get("partofset", [None])[0] + or tags.get("part_of_set", [None])[0] + or tags.get("part of set", [None])[0] + or tags.get("discnumber", [None])[0] + or tags.get("disc", [None])[0] + ) + part_of_set = _normalize_part_of_set(raw_part_of_set) return {"artist": artist, "title": title, "album": album, - "year": year, "track": track, "genre": genre} + "year": year, "track": track, "genre": genre, + "part_of_set": part_of_set} except Exception: return {"artist": None, "title": None, "album": None, - "year": None, "track": None, "genre": None} + "year": None, "track": None, "genre": None, + "part_of_set": None} def extract_primary_and_collabs(raw_artist: str): """ @@ -519,6 +559,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: year = data["year"] genre = data["genre"] track = data["track"] + part_of_set = data["part_of_set"] # 1) Primary & collabs (preserve original case) primary, collabs = extract_primary_and_collabs(raw_artist) @@ -566,6 +607,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: "year": year, "genre": genre, "track": track, + "part_of_set": part_of_set, "cover_hash": cover_hash, "folder_tags": folder_tags, "missing_core": not data.get("artist") or not data.get("title"), @@ -594,6 +636,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: album = info["album"] or "" year = info["year"] or "Unknown" track = info["track"] + part_of_set = info.get("part_of_set") folders = info["folder_tags"] if info.get("missing_core"): candidate = os.path.join(review_root, os.path.basename(old_path)) @@ -662,6 +705,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: ) artist_folder = os.path.join(MUSIC_ROOT, "By Artist", sanitize(main_artist)) base_folder = os.path.join(artist_folder, sanitize(album)) + base_folder = _apply_part_of_set(base_folder, album, part_of_set, decision_log) # Build filename (skip normal album logic) basename = os.path.basename(old_path) @@ -728,6 +772,8 @@ def _compute(path: str) -> tuple[int | None, str | None]: f"album_count={c2} ≤ 3 → group under '{primary} - Singles'" ) + base_folder = _apply_part_of_set(base_folder, album, part_of_set, decision_log) + # Build filename for all “common” tracks basename = os.path.basename(old_path) if "/" in raw_artist or is_repeated(raw_artist): @@ -752,6 +798,8 @@ def _compute(path: str) -> tuple[int | None, str | None]: leftover.discard(primary) if album: leftover.discard(album) + if part_of_set: + leftover.discard(part_of_set) leftover.discard(year) if genre: leftover.add(genre) diff --git a/music_indexer_api.py b/music_indexer_api.py index 2426ec5..449e611 100644 --- a/music_indexer_api.py +++ b/music_indexer_api.py @@ -124,16 +124,46 @@ def is_repeated(name: str) -> bool: return True return False +def _normalize_part_of_set(raw_value: str | None) -> str | None: + if not raw_value: + return None + text = str(raw_value).strip() + if not text: + return None + first = text.split("/", 1)[0].strip() + if not first: + return None + if first.isdigit(): + return f"Part {int(first)}" + return first + + +def _apply_part_of_set( + base_folder: str, + album: str, + part_of_set: str | None, + decision_log: list[str], +) -> str: + if part_of_set and album and os.path.basename(base_folder) == sanitize(album): + part_folder = sanitize(part_of_set) + decision_log.append( + f" → Part of set '{part_of_set}' detected; placed under subfolder '{part_folder}'" + ) + return os.path.join(base_folder, part_folder) + return base_folder + + def get_tags(path: str): """ - Read basic tags using Mutagen (artist, title, album, year, track, genre). + Read basic tags using Mutagen (artist, title, album, year, track, genre, part_of_set). Return a dict with those fields (or None if missing). """ try: audio = MutagenFile(path, easy=True) if not audio or not audio.tags: return {"artist": None, "title": None, "album": None, - "year": None, "track": None, "genre": None} + "year": None, "track": None, "genre": None, + "part_of_set": None} tags = audio.tags artist = tags.get("artist", [None])[0] title = tags.get("title", [None])[0] @@ -148,11 +178,21 @@ def get_tags(path: str): except Exception: track = None genre = tags.get("genre", [None])[0] + raw_part_of_set = ( + tags.get("partofset", [None])[0] + or tags.get("part_of_set", [None])[0] + or tags.get("part of set", [None])[0] + or tags.get("discnumber", [None])[0] + or tags.get("disc", [None])[0] + ) + part_of_set = _normalize_part_of_set(raw_part_of_set) return {"artist": artist, "title": title, "album": album, - "year": year, "track": track, "genre": genre} + "year": year, "track": track, "genre": genre, + "part_of_set": part_of_set} except Exception: return {"artist": None, "title": None, "album": None, - "year": None, "track": None, "genre": None} + "year": None, "track": None, "genre": None, + "part_of_set": None} def extract_primary_and_collabs(raw_artist: str): """ @@ -519,6 +559,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: year = data["year"] genre = data["genre"] track = data["track"] + part_of_set = data["part_of_set"] # 1) Primary & collabs (preserve original case) primary, collabs = extract_primary_and_collabs(raw_artist) @@ -566,6 +607,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: "year": year, "genre": genre, "track": track, + "part_of_set": part_of_set, "cover_hash": cover_hash, "folder_tags": folder_tags, "missing_core": not data.get("artist") or not data.get("title"), @@ -594,6 +636,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: album = info["album"] or "" year = info["year"] or "Unknown" track = info["track"] + part_of_set = info.get("part_of_set") folders = info["folder_tags"] if info.get("missing_core"): candidate = os.path.join(review_root, os.path.basename(old_path)) @@ -662,6 +705,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: ) artist_folder = os.path.join(MUSIC_ROOT, "By Artist", sanitize(main_artist)) base_folder = os.path.join(artist_folder, sanitize(album)) + base_folder = _apply_part_of_set(base_folder, album, part_of_set, decision_log) # Build filename (skip normal album logic) basename = os.path.basename(old_path) @@ -728,6 +772,8 @@ def _compute(path: str) -> tuple[int | None, str | None]: f"album_count={c2} ≤ 3 → group under '{primary} - Singles'" ) + base_folder = _apply_part_of_set(base_folder, album, part_of_set, decision_log) + # Build filename for all “common” tracks basename = os.path.basename(old_path) if "/" in raw_artist or is_repeated(raw_artist): @@ -752,6 +798,8 @@ def _compute(path: str) -> tuple[int | None, str | None]: leftover.discard(primary) if album: leftover.discard(album) + if part_of_set: + leftover.discard(part_of_set) leftover.discard(year) if genre: leftover.add(genre) From 87814f8aa035ec72f65274532171e8b795bb6eef Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 28 Dec 2025 20:43:36 -0500 Subject: [PATCH 194/606] Align preview report HTML with execution layout --- duplicate_consolidation.py | 760 +++++++++++++++++++++++++++++++++---- 1 file changed, 684 insertions(+), 76 deletions(-) diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index a8ea53d..2a72c7d 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -15,6 +15,7 @@ import io import json import os +import platform import re import threading import time @@ -1774,101 +1775,708 @@ def export_consolidation_preview_html(plan: ConsolidationPlan, output_html_path: def esc(value: object) -> str: return html.escape(str(value)) - summary = { - "Groups": len(plan.groups), - "Review required": plan.review_required_count, - "Placeholders present": "Yes" if plan.placeholders_present else "No", - "Generated at": plan.generated_at.isoformat(), - } + def _basename(value: str) -> str: + return os.path.basename(value) or value + + def _status_badge_class(status: str) -> str: + if status in {"success", "ready"}: + return "ok" + if status in {"failed", "blocked", "cancelled"}: + return "fail" + return "warn" + + def _format_metadata_notes(meta: Dict[str, object]) -> str: + notes = [] + for key, value in meta.items(): + if key in {"group_id", "group_ids"}: + continue + if isinstance(value, (list, tuple, set)): + rendered = ", ".join(str(v) for v in value) + else: + rendered = str(value) + notes.append(f"{key}: {rendered}") + return " | ".join(notes) + + def _format_notes(detail: str, meta_notes: str) -> str: + detail_html = html.escape(detail) + if meta_notes: + meta_html = html.escape(meta_notes) + return f"{detail_html}
      {meta_html}
      " + return detail_html + + def _planned_actions(group: GroupPlan) -> List[Dict[str, object]]: + actions: List[Dict[str, object]] = [] + for path, changes in group.metadata_changes.items(): + fields = sorted(changes.keys()) + actions.append( + { + "step": "metadata", + "target": path, + "status": "review", + "detail": "Planned metadata updates.", + "metadata": {"fields": fields}, + } + ) + for directive in group.artwork: + actions.append( + { + "step": "artwork", + "target": directive.target, + "status": "review", + "detail": f"Copy artwork from {directive.source}.", + "metadata": {"source": directive.source, "reason": directive.reason}, + } + ) + for loser in group.losers: + disposition = group.loser_disposition.get(loser, "quarantine") + actions.append( + { + "step": "loser_cleanup", + "target": loser, + "status": "review", + "detail": f"Loser marked for {disposition}.", + "metadata": {"disposition": disposition}, + } + ) + for playlist, destination in group.playlist_rewrites.items(): + actions.append( + { + "step": "playlist", + "target": playlist, + "status": "review", + "detail": "Planned playlist rewrite.", + "metadata": {"destination": destination}, + } + ) + return actions + + def _action_badges(group: GroupPlan, actions: List[Dict[str, object]]) -> List[str]: + badges = [] + meta_count = sum(1 for act in actions if act["step"] == "metadata") + art_count = sum(1 for act in actions if act["step"] == "artwork") + playlist_count = sum(1 for act in actions if act["step"] == "playlist") + loser_count = len(group.losers) + if meta_count: + badges.append(f"metadata updates ({meta_count})") + if art_count: + badges.append(f"artwork copies ({art_count})") + if playlist_count: + badges.append(f"playlist rewrites ({playlist_count})") + if loser_count: + badges.append(f"losers ({loser_count})") + return badges + + def _group_type_hint(actions: List[Dict[str, object]]) -> str: + if not actions: + return "unknown" + if all(act["step"] == "metadata" for act in actions): + return "metadata-only" + return "mixed" + + def _group_disposition_count(group: GroupPlan) -> int: + return sum( + 1 + for loser in group.losers + if group.loser_disposition.get(loser, "quarantine") != "retain" + ) + + generated_at_iso = plan.generated_at.isoformat() + host_name = platform.node() or "unknown-host" + reports_dir = os.path.dirname(output_html_path) + plan_signature = plan.plan_signature or "not-generated" + global_notes = "Groups are collapsed by default. Full paths and verbose messages are in “Context”." + if plan.review_flags: + joined_flags = "; ".join(plan.review_flags) + global_notes = f"{global_notes} Review flags: {joined_flags}" html_lines = [ "", "", "", "", - "", - "Duplicate Finder Preview", + "", + "Execution Report Preview", "", "", "", - "

      Duplicate Finder Preview

      ", - "

      Review the duplicate consolidation plan before executing.

      ", - "
      ", + "
      ", + "

      Execution Report Preview

      ", + "
      📝 review
      ", + "
      ", + "
      ", + "
      ", + "
      ", + "
      ", + "Groups 0", + "Winners 0", + "Quarantined 0", + "Ops 0/0 ok", + "
      ", + "
      ", + "", + "", + "
      ", + "
      ", + "
      ", + "
      ", + "
      Plan signature
      ", + f"
      {esc(plan_signature)}
      ", + "
      Reports directory
      ", + f"
      {esc(reports_dir)}
      ", + "
      ", + "
      ", + "
      ", + "
      ", + "
      Generated
      ", + f"
      {esc(generated_at_iso)}
      ", + "
      Host
      ", + f"
      {esc(host_name)}
      ", + "
      Notes
      ", + f"
      {esc(global_notes)}
      ", + "
      ", + "
      ", + "
      ", + "
      ", + "
      ", + "", + "", + "
      ", + "
      ", + "0 visible", + "", + "", + "
      ", + "
      ", + "
      ", ] - for label, value in summary.items(): - html_lines.append("
      ") - html_lines.append(f"
      {esc(label)}
      ") - html_lines.append(f"
      {esc(value)}
      ") - html_lines.append("
      ") - html_lines.append("
      ") - if plan.review_flags: + all_actions: List[Dict[str, object]] = [] + for group in plan.groups: + actions = _planned_actions(group) + all_actions.extend(actions) + state = "review" if group.review_flags else "ready" + badges = _action_badges(group, actions) + group_disposition_count = _group_disposition_count(group) + group_type_hint = _group_type_hint(actions) + search_tokens = [group.group_id, group.winner_path, _basename(group.winner_path), state] + search_tokens.extend(badges) + for act in actions: + search_tokens.extend([act["step"], act["target"], act["status"], act["detail"]]) + meta_notes = _format_metadata_notes(act["metadata"]) + if meta_notes: + search_tokens.append(meta_notes) + search_text = " ".join(str(token) for token in search_tokens if token) + summary_line = ( + f"Status: {state}. " + "; ".join(badges) if badges else f"Status: {state}." + ) + html_lines.append( + "
      " + ) + html_lines.append("") + html_lines.append("
      ") + html_lines.append("
      ") + html_lines.append(f"Group {esc(group.group_id)}") + html_lines.append( + "{esc(_basename(group.winner_path))}" + ) + html_lines.append("
      ") + html_lines.append("
      ") + html_lines.append( + f"" + f"{esc(state)}" + ) + for badge in badges: + html_lines.append(f"{esc(badge)}") + html_lines.append("
      ") + html_lines.append("
      ") + html_lines.append(f"
      {esc(summary_line)}
      ") + html_lines.append("
      ") + html_lines.append("
      ") html_lines.append("
      ") - html_lines.append("

      Global review flags

      ") - html_lines.append("
        ") - for flag in plan.review_flags: - html_lines.append(f"
      • {esc(flag)}
      • ") - html_lines.append("
      ") + html_lines.append("

      Context

      ") + html_lines.append("
      ") + html_lines.append("
      ") + html_lines.append("
      Kept file (winner)
      ") + html_lines.append(f"
      {esc(group.winner_path)}
      ") + html_lines.append("
      Group ID
      ") + html_lines.append(f"
      {esc(group.group_id)}
      ") + html_lines.append("
      Debug
      ") + html_lines.append( + "
      Shown for traceability; safe to ignore for normal review.
      " + ) html_lines.append("
      ") - - html_lines.extend( - [ - "
      ", - "

      Duplicate groups

      ", - "", - "", - "", - ] - ) - for group in plan.groups: - status = "Review" if group.review_flags else "Ready" - badge_class = "review" if group.review_flags else "ready" + html_lines.append("") + html_lines.append("") + html_lines.append("
      ") + html_lines.append("

      Operations

      ") + html_lines.append("
      GroupWinnerLosersStatus
      ") + html_lines.append("") html_lines.append("") - html_lines.append(f"") - html_lines.append(f"") - html_lines.append(f"") - html_lines.append(f"") + html_lines.append("") + html_lines.append("") + html_lines.append("") + html_lines.append("") html_lines.append("") - html_lines.append("
      {esc(group.group_id)}{esc(os.path.basename(group.winner_path))}{esc(len(group.losers))}{esc(status)}OperationTargetStatusNotes
      ") - - for group in plan.groups: - html_lines.append("
      ") - html_lines.append(f"Group {esc(group.group_id)}") - html_lines.append("
      ") - html_lines.append(f"Winner:\n{esc(group.winner_path)}\n") - if group.losers: - html_lines.append("Losers:") - for loser in group.losers: - html_lines.append(f"\n- {esc(loser)}") - if group.review_flags: - html_lines.append("\n\nReview flags:") - for flag in group.review_flags: - html_lines.append(f"\n- {esc(flag)}") + html_lines.append("") + html_lines.append("") + if actions: + for act in actions: + meta_notes = _format_metadata_notes(act["metadata"]) + notes = _format_notes(str(act["detail"]), meta_notes) + status_class = _status_badge_class(str(act["status"])) + html_lines.append( + "" + f"{esc(act['step'])}" + f"" + f"{esc(_basename(str(act['target'])))}" + f"{esc(act['status'])}" + f"{notes}" + "" + ) + else: + html_lines.append( + "none—" + "review" + "No planned operations for this group." + ) + html_lines.append("") + html_lines.append("") + html_lines.append("
      ") html_lines.append("
      ") html_lines.append("
      ") - html_lines.extend(["
      ", "", ""]) + html_lines.append("") + html_lines.append("
      ") + html_lines.append("") + html_lines.append("
      ") + html_lines.append( + "
      Quarantined Files (0)
      " + ) + html_lines.append("
      Collapsed by default
      ") + html_lines.append("
      ") + html_lines.append("
      ") + html_lines.append("
        ") + for group in plan.groups: + for loser in group.losers: + disposition = group.loser_disposition.get(loser, "quarantine") + if disposition == "retain": + continue + html_lines.append(f"
      • {esc(loser)} → {esc(disposition)}
      • ") + html_lines.append("
      ") + html_lines.append("
      ") + + missing_sections: List[str] = [] + metadata_planned = sum( + len(group.metadata_changes) for group in plan.groups if group.metadata_changes + ) + missing_sections.append(f"Metadata operations planned: {metadata_planned}") + planned_quarantine = sum( + 1 + for group in plan.groups + for loser in group.losers + if group.loser_disposition.get(loser, "quarantine") != "retain" + ) + missing_sections.append(f"Files quarantined: {planned_quarantine}") + + playlist_planned = sum(len(group.playlist_rewrites) for group in plan.groups) + if playlist_planned: + missing_sections.append("") + missing_sections.append("Playlist Changes") + missing_sections.append("") + missing_sections.append( + "" + ) + missing_sections.append("") + for group in plan.groups: + for playlist, destination in group.playlist_rewrites.items(): + missing_sections.append( + "" + f"" + "" + f"" + "" + ) + missing_sections.append("
      PlaylistStatusDetails
      {esc(playlist)}reviewplanned rewrite → {esc(destination)}
      ") + + if missing_sections: + html_lines.append( + "
      " + "NEED TO INCORPORATE AT LATER DATE" + "
      " + ) + for section in missing_sections: + if not section: + html_lines.append("
      ") + continue + html_lines.append(f"
      {section}
      ") + html_lines.append( + "" + ) + html_lines.append("") with open(ensure_long_path(output_html_path), "w", encoding="utf-8") as handle: handle.write("\n".join(html_lines)) From 423db849461ccee16b73f5e9069cbd12c4282979 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 28 Dec 2025 20:56:38 -0500 Subject: [PATCH 195/606] Add album art thumbnails to duplicate group headers --- duplicate_consolidation.py | 75 ++++++++++++++++++++++++++++- duplicate_consolidation_executor.py | 73 +++++++++++++++++++++++++++- 2 files changed, 144 insertions(+), 4 deletions(-) diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index 2a72c7d..35636a9 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -8,6 +8,7 @@ """ from __future__ import annotations +import base64 import datetime import html import hashlib @@ -1880,6 +1881,38 @@ def _group_disposition_count(group: GroupPlan) -> int: if group.loser_disposition.get(loser, "quarantine") != "retain" ) + def _album_art_src(group: GroupPlan) -> str | None: + chosen_hash = "" + if isinstance(group.chosen_artwork_source, Mapping): + raw_hash = group.chosen_artwork_source.get("hash") + if raw_hash: + chosen_hash = str(raw_hash) + fallback: ArtworkCandidate | None = None + for candidate in group.artwork_candidates: + if not candidate.bytes: + continue + if chosen_hash and candidate.hash == chosen_hash: + return _image_data_uri(candidate.bytes) + if fallback is None: + fallback = candidate + if fallback and fallback.bytes: + return _image_data_uri(fallback.bytes) + return None + + def _image_data_uri(payload: bytes) -> str: + mime = _image_mime(payload) + encoded = base64.b64encode(payload).decode("ascii") + return f"data:{mime};base64,{encoded}" + + def _image_mime(payload: bytes) -> str: + if payload.startswith(b"\x89PNG\r\n\x1a\n"): + return "image/png" + if payload.startswith(b"\xff\xd8"): + return "image/jpeg" + if payload[:4] == b"RIFF" and payload[8:12] == b"WEBP": + return "image/webp" + return "image/jpeg" + generated_at_iso = plan.generated_at.isoformat() host_name = platform.node() or "unknown-host" reports_dir = os.path.dirname(output_html_path) @@ -2035,18 +2068,51 @@ def _group_disposition_count(group: GroupPlan) -> int: "cursor:pointer;", "padding: 12px 12px;", "display:grid;", - "grid-template-columns: 1fr;", - "gap:8px;", + "grid-template-columns: auto 1fr;", + "grid-template-rows: auto auto;", + "column-gap: 12px;", + "row-gap: 6px;", "background: #fff;", "}", "summary.group-summary::-webkit-details-marker{ display:none; }", + ".album-art{", + "width: 42px;", + "height: 42px;", + "grid-column: 1;", + "grid-row: 1 / span 2;", + "align-self: center;", + "border-radius: 8px;", + "border: 1px solid var(--border);", + "position: relative;", + "overflow: hidden;", + "background: #f7f7f7;", + "}", + ".album-art::after{", + "content: \"\";", + "position: absolute;", + "inset: 0;", + "pointer-events: none;", + "box-shadow: inset 0 0 0 1px rgba(0,0,0,.06), inset 0 0 10px rgba(0,0,0,.20);", + "}", + ".album-art img{", + "width: 100%;", + "height: 100%;", + "object-fit: cover;", + "display: block;", + "}", ".group-top{", + "grid-column: 2;", + "grid-row: 1;", "display:flex;", "flex-wrap:wrap;", "gap:10px 12px;", "align-items:center;", "justify-content:space-between;", "}", + "summary.group-summary .tiny{", + "grid-column: 2;", + "grid-row: 2;", + "}", ".group-title{", "display:flex;", "align-items:baseline;", @@ -2248,7 +2314,12 @@ def _group_disposition_count(group: GroupPlan) -> int: f"data-search='{esc(search_text.lower())}' " f"data-quarantine-count='{group_disposition_count}'>" ) + album_art_src = _album_art_src(group) html_lines.append("") + html_lines.append("") + if album_art_src: + html_lines.append(f"") + html_lines.append("") html_lines.append("
      ") html_lines.append("
      ") html_lines.append(f"Group {esc(group.group_id)}") diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index be560bb..3db143f 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -12,6 +12,7 @@ """ from __future__ import annotations +import base64 import datetime import hashlib import html @@ -1033,6 +1034,36 @@ def _action_badges(group_actions: List[ExecutionAction]) -> List[str]: badges.append(f"artwork copied ({art_success})") return badges + def _album_art_src(group: object, winner_path: str) -> str | None: + candidates: List[str] = [] + if group is not None: + chosen = getattr(group, "chosen_artwork_source", None) + if isinstance(chosen, Mapping): + chosen_path = chosen.get("path") + if isinstance(chosen_path, str) and chosen_path: + candidates.append(chosen_path) + if winner_path: + candidates.append(winner_path) + for path in candidates: + payload = _extract_artwork_bytes(path) + if payload: + return _image_data_uri(payload) + return None + + def _image_data_uri(payload: bytes) -> str: + mime = _image_mime(payload) + encoded = base64.b64encode(payload).decode("ascii") + return f"data:{mime};base64,{encoded}" + + def _image_mime(payload: bytes) -> str: + if payload.startswith(b"\x89PNG\r\n\x1a\n"): + return "image/png" + if payload.startswith(b"\xff\xd8"): + return "image/jpeg" + if payload[:4] == b"RIFF" and payload[8:12] == b"WEBP": + return "image/webp" + return "image/jpeg" + def _format_metadata_notes(meta: Dict[str, object]) -> str: notes = [] for key, value in meta.items(): @@ -1247,18 +1278,51 @@ def _run_status() -> tuple[str, str, str]: "cursor:pointer;", "padding: 12px 12px;", "display:grid;", - "grid-template-columns: 1fr;", - "gap:8px;", + "grid-template-columns: auto 1fr;", + "grid-template-rows: auto auto;", + "column-gap: 12px;", + "row-gap: 6px;", "background: #fff;", "}", "summary.group-summary::-webkit-details-marker{ display:none; }", + ".album-art{", + "width: 42px;", + "height: 42px;", + "grid-column: 1;", + "grid-row: 1 / span 2;", + "align-self: center;", + "border-radius: 8px;", + "border: 1px solid var(--border);", + "position: relative;", + "overflow: hidden;", + "background: #f7f7f7;", + "}", + ".album-art::after{", + "content: \"\";", + "position: absolute;", + "inset: 0;", + "pointer-events: none;", + "box-shadow: inset 0 0 0 1px rgba(0,0,0,.06), inset 0 0 10px rgba(0,0,0,.20);", + "}", + ".album-art img{", + "width: 100%;", + "height: 100%;", + "object-fit: cover;", + "display: block;", + "}", ".group-top{", + "grid-column: 2;", + "grid-row: 1;", "display:flex;", "flex-wrap:wrap;", "gap:10px 12px;", "align-items:center;", "justify-content:space-between;", "}", + "summary.group-summary .tiny{", + "grid-column: 2;", + "grid-row: 2;", + "}", ".group-title{", "display:flex;", "align-items:baseline;", @@ -1464,7 +1528,12 @@ def _run_status() -> tuple[str, str, str]: f"data-search='{html.escape(search_text.lower())}' " f"data-quarantine-count='{group_quarantine_count}'>" ) + album_art_src = _album_art_src(grp, winner_path) html_lines.append("") + html_lines.append("") + if album_art_src: + html_lines.append(f"") + html_lines.append("") html_lines.append("
      ") html_lines.append("
      ") html_lines.append(f"Group {html.escape(gid)}") From ff9568091cbada5f00387a423ff9fcbeb9ac3c33 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 29 Dec 2025 17:25:46 -0500 Subject: [PATCH 196/606] Add diagnostics for plan signature mismatch --- duplicate_consolidation_executor.py | 40 +++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index 3db143f..a332b0d 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -449,11 +449,12 @@ def _detect_state_drift(expected: Mapping[str, object], actual: Mapping[str, obj def _coerce_consolidation_plan( plan_input: ConsolidationPlan | Mapping[str, object] | str, log: Callable[[str], None], -) -> tuple[ConsolidationPlan, str]: +) -> tuple[ConsolidationPlan, str, str | None]: if isinstance(plan_input, ConsolidationPlan): - return plan_input, "in-memory plan" + return plan_input, "in-memory plan", None payload: object = plan_input source = "plan payload" + plan_path: str | None = None if isinstance(plan_input, str): if os.path.exists(plan_input): if not os.path.isfile(plan_input): @@ -466,6 +467,7 @@ def _coerce_consolidation_plan( except json.JSONDecodeError as exc: raise ValueError(f"Plan JSON could not be decoded: {exc}") from exc source = f"preview output {plan_input}" + plan_path = plan_input else: try: payload = json.loads(plan_input) @@ -479,7 +481,7 @@ def _coerce_consolidation_plan( raise ValueError("Plan payload has a non-mapping 'plan' field.") payload = inner source = f"{source} (wrapped)" - return consolidation_plan_from_dict(payload), source + return consolidation_plan_from_dict(payload), source, plan_path raise ValueError("Plan input must be a ConsolidationPlan, mapping, or JSON string.") @@ -546,7 +548,7 @@ def _find_loser_references( try: try: - plan, plan_source = _coerce_consolidation_plan(plan_input, log) + plan, plan_source, plan_path = _coerce_consolidation_plan(plan_input, log) log(f"Execute plan input resolved from {plan_source}.") except ValueError as exc: success = False @@ -567,14 +569,42 @@ def _find_loser_references( _check_cancel("start") if plan.plan_signature and plan.plan_signature != computed_signature: + generated_at_iso = plan.generated_at.isoformat() + snapshot_count = len(plan.source_snapshot) + snapshot_id = hashlib.sha256( + json.dumps(plan.source_snapshot, sort_keys=True).encode("utf-8") + ).hexdigest() + log( + "Plan signature mismatch summary: " + + json.dumps( + { + "plan_path": plan_path, + "plan_source": plan_source, + "plan_signature": plan.plan_signature, + "computed_signature": computed_signature, + "generated_at": generated_at_iso, + "snapshot_id": snapshot_id, + "snapshot_count": snapshot_count, + "group_count": len(plan.groups), + }, + sort_keys=True, + ) + ) success = False _record( "preflight", "execution", "blocked", - "Plan signature mismatch; rerun preview to regenerate the plan.", + "Plan signature mismatch; rerun preview to regenerate the plan. " + "Plan signature mismatch → your preview plan is stale or not the one being executed.", expected_signature=plan.plan_signature, computed_signature=computed_signature, + plan_path=plan_path or "unknown", + plan_source=plan_source, + generated_at=generated_at_iso, + snapshot_id=snapshot_id, + snapshot_count=snapshot_count, + group_count=len(plan.groups), ) raise RuntimeError("Execution blocked: plan signature mismatch.") From 7cb3a0a63180d29a71705a958f8d214afc20d025 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 29 Dec 2025 17:38:17 -0500 Subject: [PATCH 197/606] Improve plan signature mismatch diagnostics --- duplicate_consolidation_executor.py | 61 +++++++++++++++++++++++++---- 1 file changed, 54 insertions(+), 7 deletions(-) diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index a332b0d..a718c5b 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -49,6 +49,14 @@ def _default_log(msg: str) -> None: return None +@dataclass(frozen=True) +class PlanLoadResult: + plan: ConsolidationPlan + source: str + path: str | None + input_type: str + + @dataclass class ExecutionConfig: """Configuration for consolidation execution.""" @@ -449,12 +457,13 @@ def _detect_state_drift(expected: Mapping[str, object], actual: Mapping[str, obj def _coerce_consolidation_plan( plan_input: ConsolidationPlan | Mapping[str, object] | str, log: Callable[[str], None], -) -> tuple[ConsolidationPlan, str, str | None]: +) -> PlanLoadResult: if isinstance(plan_input, ConsolidationPlan): - return plan_input, "in-memory plan", None + return PlanLoadResult(plan=plan_input, source="in-memory plan", path=None, input_type="in-memory") payload: object = plan_input source = "plan payload" plan_path: str | None = None + input_type = "mapping" if isinstance(plan_input, str): if os.path.exists(plan_input): if not os.path.isfile(plan_input): @@ -468,12 +477,14 @@ def _coerce_consolidation_plan( raise ValueError(f"Plan JSON could not be decoded: {exc}") from exc source = f"preview output {plan_input}" plan_path = plan_input + input_type = "file" else: try: payload = json.loads(plan_input) except json.JSONDecodeError as exc: raise ValueError(f"Plan JSON could not be decoded: {exc}") from exc source = "json payload" + input_type = "json string" if isinstance(payload, Mapping): if "plan" in payload: inner = payload.get("plan") @@ -481,7 +492,12 @@ def _coerce_consolidation_plan( raise ValueError("Plan payload has a non-mapping 'plan' field.") payload = inner source = f"{source} (wrapped)" - return consolidation_plan_from_dict(payload), source, plan_path + return PlanLoadResult( + plan=consolidation_plan_from_dict(payload), + source=source, + path=plan_path, + input_type=input_type, + ) raise ValueError("Plan input must be a ConsolidationPlan, mapping, or JSON string.") @@ -548,7 +564,11 @@ def _find_loser_references( try: try: - plan, plan_source, plan_path = _coerce_consolidation_plan(plan_input, log) + plan_result = _coerce_consolidation_plan(plan_input, log) + plan = plan_result.plan + plan_source = plan_result.source + plan_path = plan_result.path + plan_input_type = plan_result.input_type log(f"Execute plan input resolved from {plan_source}.") except ValueError as exc: success = False @@ -571,21 +591,45 @@ def _find_loser_references( if plan.plan_signature and plan.plan_signature != computed_signature: generated_at_iso = plan.generated_at.isoformat() snapshot_count = len(plan.source_snapshot) + canonical_snapshot = {k: dict(v) for k, v in plan.source_snapshot.items()} snapshot_id = hashlib.sha256( - json.dumps(plan.source_snapshot, sort_keys=True).encode("utf-8") + json.dumps(canonical_snapshot, sort_keys=True).encode("utf-8") ).hexdigest() + drift_examples: List[str] = [] + for path, expected in plan.source_snapshot.items(): + if len(drift_examples) >= 5: + break + actual = _capture_library_state(path) + delta = _detect_state_drift(expected, actual) + if delta: + drift_examples.append(f"{path}: {delta}") + plan_path_mtime = None + plan_path_size = None + if plan_path: + try: + plan_path_mtime = datetime.datetime.fromtimestamp( + os.path.getmtime(plan_path), datetime.timezone.utc + ).isoformat() + plan_path_size = os.path.getsize(plan_path) + except OSError: + plan_path_mtime = None + plan_path_size = None log( "Plan signature mismatch summary: " + json.dumps( { "plan_path": plan_path, "plan_source": plan_source, + "plan_input_type": plan_input_type, + "plan_path_mtime": plan_path_mtime, + "plan_path_size": plan_path_size, "plan_signature": plan.plan_signature, "computed_signature": computed_signature, "generated_at": generated_at_iso, "snapshot_id": snapshot_id, "snapshot_count": snapshot_count, "group_count": len(plan.groups), + "drift_examples": drift_examples, }, sort_keys=True, ) @@ -595,16 +639,19 @@ def _find_loser_references( "preflight", "execution", "blocked", - "Plan signature mismatch; rerun preview to regenerate the plan. " - "Plan signature mismatch → your preview plan is stale or not the one being executed.", + "Plan signature mismatch — preview plan is stale or not the one being executed. Rerun Preview.", expected_signature=plan.plan_signature, computed_signature=computed_signature, plan_path=plan_path or "unknown", plan_source=plan_source, + plan_input_type=plan_input_type, + plan_path_mtime=plan_path_mtime, + plan_path_size=plan_path_size, generated_at=generated_at_iso, snapshot_id=snapshot_id, snapshot_count=snapshot_count, group_count=len(plan.groups), + drift_examples=drift_examples, ) raise RuntimeError("Execution blocked: plan signature mismatch.") From 5fa217b56c4f868061aea514d5671bea288a494c Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 29 Dec 2025 19:15:42 -0500 Subject: [PATCH 198/606] Guard artwork extraction for report generation --- duplicate_consolidation_executor.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index a718c5b..6fb0528 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -259,12 +259,16 @@ def _extract_artwork_bytes(path: str) -> bytes | None: sidecars = [f"{path}.artwork", f"{path}.cover", f"{path}.jpg"] for candidate in sidecars: if os.path.exists(candidate): - with open(candidate, "rb") as handle: - return handle.read() + try: + with open(candidate, "rb") as handle: + return handle.read() + except OSError: + continue if MutagenFile is None: return None - audio = MutagenFile(ensure_long_path(path)) + audio = None try: + audio = MutagenFile(ensure_long_path(path)) if audio is None: return None pictures = getattr(audio, "pictures", None) @@ -278,6 +282,8 @@ def _extract_artwork_bytes(path: str) -> bytes | None: data = getattr(frame, "data", None) if data: return bytes(data) + except Exception: + return None finally: _close_mutagen_audio(audio) return None From 2d738a0a21149a7327865e3e0360b92d46e5c99c Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 30 Dec 2025 18:27:36 -0500 Subject: [PATCH 199/606] Fix plan signature refresh after plan edits --- duplicate_consolidation.py | 25 ++++++++++++++++--------- main_gui.py | 2 ++ 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index 35636a9..e73a90b 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -779,15 +779,20 @@ def __post_init__(self) -> None: snapshot[path] = dict(state) self.source_snapshot = snapshot if self.plan_signature is None: - canonical = json.dumps( - { - "generated_at": self.generated_at.isoformat(), - "groups": [g.to_dict() for g in self.groups], - "snapshot": {k: dict(v) for k, v in self.source_snapshot.items()}, - }, - sort_keys=True, - ) - self.plan_signature = hashlib.sha256(canonical.encode("utf-8")).hexdigest() + self.refresh_plan_signature() + + def refresh_plan_signature(self) -> str: + canonical = json.dumps( + { + "generated_at": self.generated_at.isoformat(), + "groups": [g.to_dict() for g in self.groups], + "snapshot": {k: dict(v) for k, v in self.source_snapshot.items()}, + }, + sort_keys=True, + ) + signature = hashlib.sha256(canonical.encode("utf-8")).hexdigest() + self.plan_signature = signature + return signature def _expect_mapping(value: object, field: str) -> Mapping[str, object]: @@ -1754,6 +1759,7 @@ def build_consolidation_plan( def export_consolidation_preview(plan: ConsolidationPlan, output_json_path: str) -> str: """Write a JSON audit of the consolidation plan.""" + plan.refresh_plan_signature() summary = { "groups": len(plan.groups), "review_required": plan.review_required_count, @@ -1773,6 +1779,7 @@ def export_consolidation_preview(plan: ConsolidationPlan, output_json_path: str) def export_consolidation_preview_html(plan: ConsolidationPlan, output_html_path: str) -> str: """Write an HTML preview of the consolidation plan.""" + plan.refresh_plan_signature() def esc(value: object) -> str: return html.escape(str(value)) diff --git a/main_gui.py b/main_gui.py index 7049167..5365e8c 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1759,6 +1759,7 @@ def _on_group_disposition_change(self, event=None) -> None: for loser in group.losers: group.loser_disposition[loser] = disposition + self._plan.refresh_plan_signature() self._render_group_details(group) self._reset_preview_if_needed() @@ -2104,6 +2105,7 @@ def _apply_deletion_mode(self) -> None: current = group.loser_disposition.get(loser, "quarantine") if current in ("retain", "quarantine", "delete"): group.loser_disposition[loser] = default_disposition + self._plan.refresh_plan_signature() self._update_groups_view(self._plan) def _reset_preview_if_needed(self) -> None: From b6608afddab722375b097f4f6696b5a6cbf9a29e Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 30 Dec 2025 18:44:06 -0500 Subject: [PATCH 200/606] Flatten duplicate quarantine output --- duplicate_consolidation_executor.py | 13 +++++++++++-- main_gui.py | 1 + 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index 6fb0528..0747acc 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -76,6 +76,7 @@ class ExecutionConfig: confirm_deletion: bool = False dry_run_execute: bool = False retain_losers: bool = False + quarantine_flatten: bool = False @dataclass @@ -423,7 +424,10 @@ def _close_mutagen_audio(audio: object | None) -> None: pass -def _quarantine_path(path: str, quarantine_root: str, library_root: str) -> str: +def _quarantine_path(path: str, quarantine_root: str, library_root: str, *, flatten: bool) -> str: + if flatten: + os.makedirs(quarantine_root, exist_ok=True) + return os.path.join(quarantine_root, os.path.basename(path)) try: rel = os.path.relpath(path, library_root) if rel.startswith(".."): @@ -986,7 +990,12 @@ def _find_loser_references( ) raise else: - dest = _quarantine_path(loser, quarantine_dir, config.library_root) + dest = _quarantine_path( + loser, + quarantine_dir, + config.library_root, + flatten=config.quarantine_flatten, + ) try: if os.path.exists(loser): shutil.move(loser, dest) diff --git a/main_gui.py b/main_gui.py index 5365e8c..e9e69f9 100644 --- a/main_gui.py +++ b/main_gui.py @@ -2002,6 +2002,7 @@ def log_callback(msg: str) -> None: reports_dir=reports_dir, playlists_dir=playlists_dir, quarantine_dir=os.path.join(path, "Quarantine"), + quarantine_flatten=True, log_callback=log_callback, allow_review_required=self.override_review_var.get(), retain_losers=not self.quarantine_var.get(), From 957bf9fe5408930257122591137eaf6fbbc67b93 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 30 Dec 2025 21:13:14 -0500 Subject: [PATCH 201/606] Unify duplicate groups by audio identity --- duplicate_consolidation.py | 545 +++++++++++++++++-------------------- 1 file changed, 243 insertions(+), 302 deletions(-) diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index e73a90b..2cb2701 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -189,22 +189,6 @@ def _album_display_label(tags: Mapping[str, object]) -> str: return album or artist or "Unknown album" -def _build_album_track_counts(tracks: Sequence["DuplicateTrack"]) -> Dict[tuple[str, str], int]: - album_titles: Dict[tuple[str, str], set[str]] = defaultdict(set) - for track in tracks: - context, _ = _classify_context(track) - if context == "unknown": - continue - tags = track.current_tags if isinstance(track.current_tags, Mapping) else track.tags - tags = tags or {} - album_key = _normalize_album_key(tags, track.path) - if not album_key: - continue - title_key = _normalize_title(tags, track.path) - album_titles[album_key].add(title_key) - return {key: len(titles) for key, titles in album_titles.items()} - - def _metadata_bucket_key(track: DuplicateTrack) -> tuple[str, str]: tags = track.current_tags if isinstance(track.current_tags, Mapping) else track.tags tags = tags or {} @@ -1418,8 +1402,6 @@ def build_consolidation_plan( review_flags.append(f"Truncated candidate set to {max_candidates} items to protect runtime.") break - album_track_counts = _build_album_track_counts(normalized) - clusters = _cluster_duplicates( normalized, exact_duplicate_threshold=exact_duplicate_threshold, @@ -1444,311 +1426,270 @@ def build_consolidation_plan( t.context_evidence = evidence context_evidence[t.path] = evidence - album_key_by_path: Dict[str, tuple[str, str] | None] = {} - release_by_path: Dict[str, str] = {} album_label_by_key: Dict[tuple[str, str], str] = {} + album_keys: set[tuple[str, str]] = set() for t in cluster_tracks: tags = t.current_tags if isinstance(t.current_tags, Mapping) else t.tags tags = tags or {} album_key = _normalize_album_key(tags, t.path) - album_key_by_path[t.path] = album_key - if album_key and album_key not in album_label_by_key: - album_label_by_key[album_key] = _album_display_label(tags) - context = contexts.get(t.path, "unknown") - if context == "unknown" or not album_key: - release_by_path[t.path] = "unknown" - continue - album_count = album_track_counts.get(album_key, 0) - if album_count >= 2: - release_by_path[t.path] = "album" - elif album_count == 1: - release_by_path[t.path] = "single" - else: - release_by_path[t.path] = "unknown" - - album_groups: Dict[tuple[str, str], List[DuplicateTrack]] = defaultdict(list) - single_tracks: List[DuplicateTrack] = [] - unknown_tracks: List[DuplicateTrack] = [] - for t in cluster_tracks: - release_type = release_by_path.get(t.path, "unknown") - if release_type == "album" and album_key_by_path.get(t.path): - album_groups[album_key_by_path[t.path]].append(t) - elif release_type == "single": - single_tracks.append(t) - else: - unknown_tracks.append(t) - - release_groups: List[tuple[List[DuplicateTrack], List[str]]] = [] - if len(album_groups) > 1: - album_labels = [ - album_label_by_key.get(key, "Unknown album") for key in sorted(album_groups.keys()) - ] + if album_key: + album_keys.add(album_key) + if album_key not in album_label_by_key: + album_label_by_key[album_key] = _album_display_label(tags) + + release_notes: List[str] = [] + if len(album_keys) > 1: + album_labels = [album_label_by_key.get(key, "Unknown album") for key in sorted(album_keys)] album_summary = ", ".join(sorted({label for label in album_labels if label})) - cross_album_note = "Identical recording across albums; preserving tracks for distinct releases." + cross_album_note = "Identical recording across albums; consolidated into one canonical track." if album_summary: cross_album_note = f"{cross_album_note} Albums: {album_summary}." - for album_key, tracks in sorted( - album_groups.items(), key=lambda item: album_label_by_key.get(item[0], "") - ): - if tracks: - release_groups.append((tracks, [cross_album_note])) - if single_tracks: - release_groups.append( - ( - single_tracks, - [ - cross_album_note, - "Singles consolidated separately from multi-album matches.", - ], - ) - ) - if unknown_tracks: - release_groups.append((unknown_tracks, [cross_album_note, "Unknown release context kept separate."])) - else: - release_groups.append((cluster_tracks, [])) - - for group_tracks, release_notes in release_groups: - group_paths = {t.path for t in group_tracks} - group_contexts = {path: contexts[path] for path in group_paths} - group_context_evidence = {path: context_evidence[path] for path in group_paths} - group_decisions = [ - d for d in cluster.decisions if d.anchor_path in group_paths and d.candidate_path in group_paths - ] - - quality_sorted = sorted(group_tracks, key=lambda t: _quality_tuple(t, group_contexts[t.path]), reverse=True) - winner = quality_sorted[0] - losers = [t.path for t in quality_sorted[1:]] - runner_up = quality_sorted[1] if len(quality_sorted) > 1 else None - - metadata_source = _select_metadata_source(group_tracks, group_contexts) - planned_tags, tag_source, tag_source_reason, tag_source_evidence = _build_planned_tags( - winner, metadata_source, group_contexts - ) - group_review: List[str] = list(release_notes) - if not metadata_source: - review_flags.append(f"Missing metadata source for group containing {winner.path}.") - group_review.append("Metadata source missing; tags may be incomplete.") - - meta_changes = _metadata_changes(winner, planned_tags) - winner_context = group_contexts.get(winner.path, "unknown") - winner_quality = _quality_rationale(winner, runner_up, winner_context) - - distance_samples: List[float] = [] - pair_distances: Dict[str, Dict[str, float]] = {t.path: {} for t in group_tracks} - missing_fingerprints = any(not t.fingerprint for t in group_tracks) - for idx, t in enumerate(group_tracks): - for other in group_tracks[idx + 1 :]: - dist = fingerprint_distance(t.fingerprint, other.fingerprint) - distance_samples.append(dist) - pair_distances[t.path][other.path] = dist - pair_distances[other.path][t.path] = dist - max_distance = max(distance_samples) if distance_samples else (1.0 if missing_fingerprints else 0.0) - - track_quality: Dict[str, Dict[str, object]] = { - t.path: { - "container": t.container or t.ext.replace(".", "").upper(), - "codec": t.codec, - "bitrate": t.bitrate, - "sample_rate": t.sample_rate, - "bit_depth": t.bit_depth, - "channels": t.channels, - "is_lossless": t.is_lossless, - } - for t in quality_sorted + release_notes.append(cross_album_note) + if len({contexts[path] for path in contexts}) > 1: + release_notes.append("Cluster spans multiple release contexts; using one canonical winner across variants.") + + group_tracks = cluster_tracks + group_paths = {t.path for t in group_tracks} + group_contexts = {path: contexts[path] for path in group_paths} + group_context_evidence = {path: context_evidence[path] for path in group_paths} + group_decisions = [ + d for d in cluster.decisions if d.anchor_path in group_paths and d.candidate_path in group_paths + ] + + quality_sorted = sorted(group_tracks, key=lambda t: _quality_tuple(t, group_contexts[t.path]), reverse=True) + winner = quality_sorted[0] + losers = [t.path for t in quality_sorted[1:]] + runner_up = quality_sorted[1] if len(quality_sorted) > 1 else None + + metadata_source = _select_metadata_source(group_tracks, group_contexts) + planned_tags, tag_source, tag_source_reason, tag_source_evidence = _build_planned_tags( + winner, metadata_source, group_contexts + ) + group_review: List[str] = list(release_notes) + if not metadata_source: + review_flags.append(f"Missing metadata source for group containing {winner.path}.") + group_review.append("Metadata source missing; tags may be incomplete.") + + meta_changes = _metadata_changes(winner, planned_tags) + winner_context = group_contexts.get(winner.path, "unknown") + winner_quality = _quality_rationale(winner, runner_up, winner_context) + + distance_samples: List[float] = [] + pair_distances: Dict[str, Dict[str, float]] = {t.path: {} for t in group_tracks} + missing_fingerprints = any(not t.fingerprint for t in group_tracks) + for idx, t in enumerate(group_tracks): + for other in group_tracks[idx + 1 :]: + dist = fingerprint_distance(t.fingerprint, other.fingerprint) + distance_samples.append(dist) + pair_distances[t.path][other.path] = dist + pair_distances[other.path][t.path] = dist + max_distance = max(distance_samples) if distance_samples else (1.0 if missing_fingerprints else 0.0) + + track_quality: Dict[str, Dict[str, object]] = { + t.path: { + "container": t.container or t.ext.replace(".", "").upper(), + "codec": t.codec, + "bitrate": t.bitrate, + "sample_rate": t.sample_rate, + "bit_depth": t.bit_depth, + "channels": t.channels, + "is_lossless": t.is_lossless, } + for t in quality_sorted + } - single_art_track, single_art_blob, ambiguous_single_art = _select_single_artwork_candidate( - group_tracks, group_contexts + single_art_track, single_art_blob, ambiguous_single_art = _select_single_artwork_candidate( + group_tracks, group_contexts + ) + overall_art_track, overall_art_blob, ambiguous_overall_art = _select_overall_artwork_candidate( + group_tracks + ) + artwork_actions: List[ArtworkDirective] = [] + chosen_artwork_source: Dict[str, object] = {"path": None, "reason": ""} + artwork_status = "unchanged" + artwork_evidence: List[str] = [] + + if single_art_blob: + chosen_artwork_source = { + "path": single_art_track.path if single_art_track else None, + "hash": single_art_blob.hash, + "size": single_art_blob.size, + "width": single_art_blob.width, + "height": single_art_blob.height, + "context": "single", + "reason": "Best single artwork by resolution/size", + } + artwork_evidence.append( + f"Single artwork selected from {single_art_track.path if single_art_track else 'unknown'} " + f"({single_art_blob.width}x{single_art_blob.height}, {single_art_blob.size} bytes)" ) - overall_art_track, overall_art_blob, ambiguous_overall_art = _select_overall_artwork_candidate( - group_tracks + elif overall_art_blob: + chosen_artwork_source = { + "path": overall_art_track.path if overall_art_track else None, + "hash": overall_art_blob.hash, + "size": overall_art_blob.size, + "width": overall_art_blob.width, + "height": overall_art_blob.height, + "context": "fallback", + "reason": "No single artwork found; best available artwork", + } + artwork_evidence.append( + f"No single artwork found; using best available artwork from {overall_art_track.path if overall_art_track else 'unknown'} " + f"({overall_art_blob.width}x{overall_art_blob.height}, {overall_art_blob.size} bytes)" ) - artwork_actions: List[ArtworkDirective] = [] - chosen_artwork_source: Dict[str, object] = {"path": None, "reason": ""} - artwork_status = "unchanged" - artwork_evidence: List[str] = [] - - if single_art_blob: - chosen_artwork_source = { - "path": single_art_track.path if single_art_track else None, - "hash": single_art_blob.hash, - "size": single_art_blob.size, - "width": single_art_blob.width, - "height": single_art_blob.height, - "context": "single", - "reason": "Best single artwork by resolution/size", - } - artwork_evidence.append( - f"Single artwork selected from {single_art_track.path if single_art_track else 'unknown'} " - f"({single_art_blob.width}x{single_art_blob.height}, {single_art_blob.size} bytes)" - ) - elif overall_art_blob: - chosen_artwork_source = { - "path": overall_art_track.path if overall_art_track else None, - "hash": overall_art_blob.hash, - "size": overall_art_blob.size, - "width": overall_art_blob.width, - "height": overall_art_blob.height, - "context": "fallback", - "reason": "No single artwork found; best available artwork", - } - artwork_evidence.append( - f"No single artwork found; using best available artwork from {overall_art_track.path if overall_art_track else 'unknown'} " - f"({overall_art_blob.width}x{overall_art_blob.height}, {overall_art_blob.size} bytes)" - ) - else: - chosen_artwork_source["reason"] = "No artwork candidates available" - artwork_status = "none found" - artwork_evidence.append("No readable artwork candidates were discovered.") - - if winner_context == "album" and single_art_blob and single_art_track: - if not winner.cover_hash or winner.cover_hash != single_art_blob.hash: - artwork_status = "apply" - artwork_actions.append( - ArtworkDirective( - source=single_art_track.path, - target=winner.path, - reason="Apply single artwork to album-context winner", - ) - ) - artwork_evidence.append("Album-context winner will receive single artwork to preserve single cover.") - else: - artwork_status = "unchanged" - chosen_artwork_source["reason"] = "Winner already has selected single artwork" - artwork_evidence.append("Winner already matches selected single artwork hash; no copy needed.") - elif winner_context == "album" and not single_art_blob and overall_art_blob and overall_art_track: - if not winner.cover_hash or winner.cover_hash != overall_art_blob.hash: - artwork_status = "apply" - artwork_actions.append( - ArtworkDirective( - source=overall_art_track.path, - target=winner.path, - reason="Fill missing album artwork from best available source", - ) + else: + chosen_artwork_source["reason"] = "No artwork candidates available" + artwork_status = "none found" + artwork_evidence.append("No readable artwork candidates were discovered.") + + if winner_context == "album" and single_art_blob and single_art_track: + if not winner.cover_hash or winner.cover_hash != single_art_blob.hash: + artwork_status = "apply" + artwork_actions.append( + ArtworkDirective( + source=single_art_track.path, + target=winner.path, + reason="Apply single artwork to album-context winner", ) - artwork_evidence.append("No single artwork available; applying best available artwork to album winner.") - else: - artwork_evidence.append("Album winner artwork already matches best available source.") + ) + artwork_evidence.append("Album-context winner will receive single artwork to preserve single cover.") else: - if winner_context == "single": - artwork_evidence.append("Single-context winner retains its own artwork; album art will not be pulled.") - chosen_artwork_source["reason"] = ( - chosen_artwork_source.get("reason") or "Single winner keeps artwork" + artwork_status = "unchanged" + chosen_artwork_source["reason"] = "Winner already has selected single artwork" + artwork_evidence.append("Winner already matches selected single artwork hash; no copy needed.") + elif winner_context == "album" and not single_art_blob and overall_art_blob and overall_art_track: + if not winner.cover_hash or winner.cover_hash != overall_art_blob.hash: + artwork_status = "apply" + artwork_actions.append( + ArtworkDirective( + source=overall_art_track.path, + target=winner.path, + reason="Fill missing album artwork from best available source", ) - elif not single_art_blob and not overall_art_blob: - group_review.append("No artwork available to apply.") - - cover_hashes = {t.cover_hash for t in group_tracks if t.cover_hash} - ambiguous_art = ambiguous_single_art or ambiguous_overall_art - if len(cover_hashes) > 1: - ambiguous_art = True - artwork_evidence.append("Conflicting embedded artwork hashes between candidates.") - if ambiguous_art: - review_flags.append( - f"Artwork selection ambiguous for group {_stable_group_id([t.path for t in group_tracks])}." ) - if not (single_art_blob or overall_art_blob): - missing_reason = "No artwork candidates available" - if any(t.artwork_error for t in group_tracks): - missing_reason = "Artwork present but unreadable" - review_flags.append(f"{missing_reason} for group {_stable_group_id([t.path for t in group_tracks])}.") - chosen_artwork_source["reason"] = missing_reason - artwork_status = "none found" - group_review.append(missing_reason) - - required_tag_gaps = [key for key in ("artist", "title") if not planned_tags.get(key)] - if required_tag_gaps: - group_review.append(f"Missing critical tags: {', '.join(sorted(required_tag_gaps))}.") - - group_confidence = "High (identical fingerprint cluster)" - if missing_fingerprints: - group_confidence = "Low (missing fingerprints in cluster)" - group_review.append("One or more tracks missing fingerprints; requires review.") - near_distances = [d for d in distance_samples if d > exact_duplicate_threshold] - if near_distances: - group_confidence = f"Medium (near-duplicate fingerprint distance up to {max_distance:.3f})" - group_review.append( - f"Audio match requires review (max distance {max_distance:.3f}; near-duplicate threshold {near_duplicate_threshold:.3f})." + artwork_evidence.append("No single artwork available; applying best available artwork to album winner.") + else: + artwork_evidence.append("Album winner artwork already matches best available source.") + else: + if winner_context == "single": + artwork_evidence.append("Single-context winner retains its own artwork; album art will not be pulled.") + chosen_artwork_source["reason"] = ( + chosen_artwork_source.get("reason") or "Single winner keeps artwork" ) - if max_distance > near_duplicate_threshold: - group_confidence = f"Low (max fingerprint distance {max_distance:.3f})" - if not missing_fingerprints: - group_review.append( - f"Fingerprint distances exceed near-duplicate threshold ({near_duplicate_threshold:.3f}); grouping may be unsafe." - ) - - dispositions = {loser: "quarantine" for loser in losers} - playlist_map = {loser: winner.path for loser in losers} - - group_id = _stable_group_id([t.path for t in group_tracks]) - if ambiguous_art: - group_review.append("Artwork selection requires review.") - if not losers: - group_review.append("No losers to consolidate.") - if any(_has_review_keyword(t) for t in group_tracks): - group_review.append("Contains remix/sped-up variant indicators; review recommended.") - placeholders = _placeholder_present(planned_tags) or any( - _placeholder_present(t.current_tags) for t in group_tracks + elif not single_art_blob and not overall_art_blob: + group_review.append("No artwork available to apply.") + + cover_hashes = {t.cover_hash for t in group_tracks if t.cover_hash} + ambiguous_art = ambiguous_single_art or ambiguous_overall_art + if len(cover_hashes) > 1: + ambiguous_art = True + artwork_evidence.append("Conflicting embedded artwork hashes between candidates.") + if ambiguous_art: + review_flags.append( + f"Artwork selection ambiguous for group {_stable_group_id([t.path for t in group_tracks])}." ) - if placeholders: - group_review.append("Placeholder metadata detected; requires review.") - plan_placeholders = True - if winner.metadata_error: - group_review.append(f"Metadata read issue for winner: {winner.metadata_error}") + if not (single_art_blob or overall_art_blob): + missing_reason = "No artwork candidates available" if any(t.artwork_error for t in group_tracks): - group_review.append("Artwork extraction failed for at least one track.") - - playlist_impact = PlaylistImpact(playlists=len(losers), entries=len(losers)) - track_states = {t.path: dict(t.library_state) for t in group_tracks} - - artwork_candidates: List[ArtworkCandidate] = [] - for t in group_tracks: - artwork_candidates.extend(t.artwork) - - current_tags = {t.path: _merge_tags(_blank_tags(), t.current_tags) for t in group_tracks} - - plans.append( - GroupPlan( - group_id=group_id, - winner_path=winner.path, - losers=losers, - planned_winner_tags=planned_tags, - winner_current_tags=_merge_tags(_blank_tags(), winner.current_tags), - current_tags=current_tags, - metadata_changes=meta_changes, - winner_quality=winner_quality, - artwork=artwork_actions, - artwork_candidates=artwork_candidates, - chosen_artwork_source=chosen_artwork_source, - artwork_status=artwork_status, - loser_disposition=dispositions, - playlist_rewrites=playlist_map, - playlist_impact=playlist_impact, - review_flags=group_review, - context_summary={ - "album": sorted([p for p in group_paths if group_contexts.get(p) == "album"]), - "single": sorted([p for p in group_paths if group_contexts.get(p) == "single"]), - "unknown": sorted([p for p in group_paths if group_contexts.get(p) == "unknown"]), - }, - context_evidence=group_context_evidence, - tag_source=tag_source, - placeholders_present=placeholders, - tag_source_reason=tag_source_reason, - tag_source_evidence=tag_source_evidence, - track_quality=track_quality, - group_confidence=group_confidence, - group_match_type="Exact" if max_distance <= exact_duplicate_threshold else "Near-duplicate", - grouping_metadata_key=cluster.metadata_key, - grouping_thresholds={ - "exact": cluster.exact_threshold, - "near": cluster.near_threshold, - }, - grouping_decisions=group_decisions, - artwork_evidence=artwork_evidence, - fingerprint_distances=pair_distances, - library_state=track_states, + missing_reason = "Artwork present but unreadable" + review_flags.append(f"{missing_reason} for group {_stable_group_id([t.path for t in group_tracks])}.") + chosen_artwork_source["reason"] = missing_reason + artwork_status = "none found" + group_review.append(missing_reason) + + required_tag_gaps = [key for key in ("artist", "title") if not planned_tags.get(key)] + if required_tag_gaps: + group_review.append(f"Missing critical tags: {', '.join(sorted(required_tag_gaps))}.") + + group_confidence = "High (identical fingerprint cluster)" + if missing_fingerprints: + group_confidence = "Low (missing fingerprints in cluster)" + group_review.append("One or more tracks missing fingerprints; requires review.") + near_distances = [d for d in distance_samples if d > exact_duplicate_threshold] + if near_distances: + group_confidence = f"Medium (near-duplicate fingerprint distance up to {max_distance:.3f})" + group_review.append( + f"Audio match requires review (max distance {max_distance:.3f}; near-duplicate threshold {near_duplicate_threshold:.3f})." + ) + if max_distance > near_duplicate_threshold: + group_confidence = f"Low (max fingerprint distance {max_distance:.3f})" + if not missing_fingerprints: + group_review.append( + f"Fingerprint distances exceed near-duplicate threshold ({near_duplicate_threshold:.3f}); grouping may be unsafe." ) + + dispositions = {loser: "quarantine" for loser in losers} + playlist_map = {loser: winner.path for loser in losers} + + group_id = _stable_group_id([t.path for t in group_tracks]) + if ambiguous_art: + group_review.append("Artwork selection requires review.") + if not losers: + group_review.append("No losers to consolidate.") + if any(_has_review_keyword(t) for t in group_tracks): + group_review.append("Contains remix/sped-up variant indicators; review recommended.") + placeholders = _placeholder_present(planned_tags) or any( + _placeholder_present(t.current_tags) for t in group_tracks + ) + if placeholders: + group_review.append("Placeholder metadata detected; requires review.") + plan_placeholders = True + if winner.metadata_error: + group_review.append(f"Metadata read issue for winner: {winner.metadata_error}") + if any(t.artwork_error for t in group_tracks): + group_review.append("Artwork extraction failed for at least one track.") + + playlist_impact = PlaylistImpact(playlists=len(losers), entries=len(losers)) + track_states = {t.path: dict(t.library_state) for t in group_tracks} + + artwork_candidates: List[ArtworkCandidate] = [] + for t in group_tracks: + artwork_candidates.extend(t.artwork) + + current_tags = {t.path: _merge_tags(_blank_tags(), t.current_tags) for t in group_tracks} + + plans.append( + GroupPlan( + group_id=group_id, + winner_path=winner.path, + losers=losers, + planned_winner_tags=planned_tags, + winner_current_tags=_merge_tags(_blank_tags(), winner.current_tags), + current_tags=current_tags, + metadata_changes=meta_changes, + winner_quality=winner_quality, + artwork=artwork_actions, + artwork_candidates=artwork_candidates, + chosen_artwork_source=chosen_artwork_source, + artwork_status=artwork_status, + loser_disposition=dispositions, + playlist_rewrites=playlist_map, + playlist_impact=playlist_impact, + review_flags=group_review, + context_summary={ + "album": sorted([p for p in group_paths if group_contexts.get(p) == "album"]), + "single": sorted([p for p in group_paths if group_contexts.get(p) == "single"]), + "unknown": sorted([p for p in group_paths if group_contexts.get(p) == "unknown"]), + }, + context_evidence=group_context_evidence, + tag_source=tag_source, + placeholders_present=placeholders, + tag_source_reason=tag_source_reason, + tag_source_evidence=tag_source_evidence, + track_quality=track_quality, + group_confidence=group_confidence, + group_match_type="Exact" if max_distance <= exact_duplicate_threshold else "Near-duplicate", + grouping_metadata_key=cluster.metadata_key, + grouping_thresholds={ + "exact": cluster.exact_threshold, + "near": cluster.near_threshold, + }, + grouping_decisions=group_decisions, + artwork_evidence=artwork_evidence, + fingerprint_distances=pair_distances, + library_state=track_states, ) + ) if plan_placeholders and "Placeholder metadata detected; review required." not in review_flags: review_flags.append("Placeholder metadata detected; review required.") From b3ba81c5703ee9d7a6620ec3d26e2d46498a08b0 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 30 Dec 2025 21:22:09 -0500 Subject: [PATCH 202/606] Handle single-release artwork migration --- duplicate_consolidation.py | 105 +++++++++++++++++++++++----- duplicate_consolidation_executor.py | 4 +- 2 files changed, 89 insertions(+), 20 deletions(-) diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index 2cb2701..c40b455 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -133,6 +133,47 @@ def _parse_int(value: object) -> int | None: return None +def _parse_track_total(value: object) -> int | None: + if isinstance(value, list): + return _parse_track_total(value[0]) if value else None + if isinstance(value, tuple): + return _parse_track_total(value[0]) if value else None + if value is None: + return None + text = str(value) + if "/" in text: + parts = text.split("/", 1) + if len(parts) == 2 and parts[1].strip(): + return _parse_int(parts[1]) + return None + + +def _release_track_total(tags: Mapping[str, object]) -> int | None: + for key in ("track", "tracknumber"): + total = _parse_track_total(tags.get(key)) + if total is not None: + return total + for key in ("tracktotal", "totaltracks", "track_total", "total_tracks"): + total = _parse_int(tags.get(key)) + if total is not None: + return total + return None + + +def _release_size_context(tags: Mapping[str, object]) -> tuple[str, str]: + album_type = str(tags.get("album_type") or tags.get("release_type") or "").lower() + track_total = _release_track_total(tags) + if track_total == 1: + return "single", "Track total indicates 1" + if track_total and track_total >= 2: + return "multi", f"Track total indicates {track_total}" + if album_type == "single": + return "single", "Album type tag indicates single" + if album_type in {"album", "lp", "ep"}: + return "multi", f"Album type tag indicates {album_type}" + return "unknown", "No track-total or album-type signal" + + def _normalized_tokens(text: str) -> List[str]: lowered = (text or "").lower() cleaned = re.sub(r"[\\[\\]{}()]+", " ", lowered) @@ -1202,15 +1243,15 @@ def _best_artwork_blob(candidates: Sequence[ArtworkCandidate]) -> ArtworkCandida return sorted(usable, key=lambda c: (_artwork_blob_score(c), c.hash), reverse=True)[0] -def _select_single_artwork_candidate( - candidates: Sequence[DuplicateTrack], contexts: Mapping[str, str] +def _select_single_release_artwork_candidate( + candidates: Sequence[DuplicateTrack], release_sizes: Mapping[str, str] ) -> tuple[DuplicateTrack | None, ArtworkCandidate | None, bool]: best_track: DuplicateTrack | None = None best_blob: ArtworkCandidate | None = None best_score: tuple | None = None ambiguous = False for track in candidates: - if contexts.get(track.path) != "single" or not track.artwork: + if release_sizes.get(track.path) != "single" or not track.artwork: continue blob = _best_artwork_blob(track.artwork) if not blob: @@ -1420,11 +1461,18 @@ def build_consolidation_plan( cluster_tracks = cluster.tracks contexts: Dict[str, str] = {} context_evidence: Dict[str, List[str]] = {} + release_sizes: Dict[str, str] = {} + release_size_evidence: Dict[str, str] = {} for t in cluster_tracks: ctx, evidence = _classify_context(t) contexts[t.path] = ctx t.context_evidence = evidence context_evidence[t.path] = evidence + tags = t.current_tags if isinstance(t.current_tags, Mapping) else t.tags + tags = tags or {} + release_size, release_reason = _release_size_context(tags) + release_sizes[t.path] = release_size + release_size_evidence[t.path] = release_reason album_label_by_key: Dict[tuple[str, str], str] = {} album_keys: set[tuple[str, str]] = set() @@ -1498,8 +1546,8 @@ def build_consolidation_plan( for t in quality_sorted } - single_art_track, single_art_blob, ambiguous_single_art = _select_single_artwork_candidate( - group_tracks, group_contexts + single_art_track, single_art_blob, ambiguous_single_art = _select_single_release_artwork_candidate( + group_tracks, release_sizes ) overall_art_track, overall_art_blob, ambiguous_overall_art = _select_overall_artwork_candidate( group_tracks @@ -1508,6 +1556,12 @@ def build_consolidation_plan( chosen_artwork_source: Dict[str, object] = {"path": None, "reason": ""} artwork_status = "unchanged" artwork_evidence: List[str] = [] + winner_release = release_sizes.get(winner.path, "unknown") + winner_multi_release = winner_release == "multi" + if winner_release != "unknown": + artwork_evidence.append( + f"Winner release size classified as {winner_release} ({release_size_evidence.get(winner.path)})." + ) if single_art_blob: chosen_artwork_source = { @@ -1516,13 +1570,17 @@ def build_consolidation_plan( "size": single_art_blob.size, "width": single_art_blob.width, "height": single_art_blob.height, - "context": "single", - "reason": "Best single artwork by resolution/size", + "context": "single-release", + "reason": "Best single-release artwork by resolution/size", } artwork_evidence.append( - f"Single artwork selected from {single_art_track.path if single_art_track else 'unknown'} " + f"Single-release artwork selected from {single_art_track.path if single_art_track else 'unknown'} " f"({single_art_blob.width}x{single_art_blob.height}, {single_art_blob.size} bytes)" ) + if single_art_track: + artwork_evidence.append( + f"Single-release classification: {release_size_evidence.get(single_art_track.path)}." + ) elif overall_art_blob: chosen_artwork_source = { "path": overall_art_track.path if overall_art_track else None, @@ -1542,32 +1600,43 @@ def build_consolidation_plan( artwork_status = "none found" artwork_evidence.append("No readable artwork candidates were discovered.") - if winner_context == "album" and single_art_blob and single_art_track: + if winner_multi_release and single_art_blob and single_art_track: if not winner.cover_hash or winner.cover_hash != single_art_blob.hash: artwork_status = "apply" artwork_actions.append( ArtworkDirective( source=single_art_track.path, target=winner.path, - reason="Apply single artwork to album-context winner", + reason="Migrate single-release artwork onto multi-track album winner", ) ) - artwork_evidence.append("Album-context winner will receive single artwork to preserve single cover.") + artwork_evidence.append( + "Multi-track album winner will receive single-release artwork to preserve unique single cover." + ) else: artwork_status = "unchanged" - chosen_artwork_source["reason"] = "Winner already has selected single artwork" - artwork_evidence.append("Winner already matches selected single artwork hash; no copy needed.") - elif winner_context == "album" and not single_art_blob and overall_art_blob and overall_art_track: - if not winner.cover_hash or winner.cover_hash != overall_art_blob.hash: + chosen_artwork_source["reason"] = "Winner already has selected single-release artwork" + artwork_evidence.append("Winner already matches selected single-release artwork hash; no copy needed.") + elif winner_multi_release and not single_art_blob and overall_art_blob and overall_art_track: + source_release = release_sizes.get(overall_art_track.path, "unknown") + if source_release == "multi" and overall_art_track.path != winner.path: + artwork_status = "unchanged" + chosen_artwork_source["reason"] = "Skipped multi-track artwork migration to avoid cross-album contamination" + artwork_evidence.append( + "Skipping artwork migration between multi-track releases to avoid cross-album contamination." + ) + elif not winner.cover_hash or winner.cover_hash != overall_art_blob.hash: artwork_status = "apply" artwork_actions.append( ArtworkDirective( source=overall_art_track.path, target=winner.path, - reason="Fill missing album artwork from best available source", + reason="Fill missing artwork from best available source", ) ) - artwork_evidence.append("No single artwork available; applying best available artwork to album winner.") + artwork_evidence.append( + "No single-release artwork available; applying best available artwork to album winner." + ) else: artwork_evidence.append("Album winner artwork already matches best available source.") else: @@ -1772,7 +1841,7 @@ def _planned_actions(group: GroupPlan) -> List[Dict[str, object]]: "step": "artwork", "target": directive.target, "status": "review", - "detail": f"Copy artwork from {directive.source}.", + "detail": f"Migrate artwork from {directive.source} to {directive.target}.", "metadata": {"source": directive.source, "reason": directive.reason}, } ) diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index 0747acc..5b54a7f 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -891,7 +891,7 @@ def _find_loser_references( "artwork", art.target, "skipped", - "Dry-run execute enabled; artwork not applied.", + f"Dry-run execute enabled; artwork not applied (source: {art.source}).", source=art.source, group_id=path_to_group.get(art.target), ) @@ -906,7 +906,7 @@ def _find_loser_references( "artwork", art.target, status, - detail or art.reason, + f"{detail or art.reason} (source: {art.source})", source=art.source, group_id=path_to_group.get(art.target), ) From 545699a194983e5b982b72169b27342977bbf9f8 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 30 Dec 2025 22:23:07 -0500 Subject: [PATCH 203/606] Add loser context card to duplicate report --- duplicate_consolidation.py | 53 +++++++++++++++++++++++++++ duplicate_consolidation_executor.py | 56 +++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+) diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index c40b455..dc89a03 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -1993,6 +1993,10 @@ def _image_mime(payload: bytes) -> str: "padding: 12px;", "background: var(--card);", "}", + ".card-stack{", + "display:grid;", + "gap:10px;", + "}", ".summary{", "display:grid;", "grid-template-columns: 1fr;", @@ -2104,6 +2108,11 @@ def _image_mime(payload: bytes) -> str: "overflow: hidden;", "background: #f7f7f7;", "}", + ".album-art.album-art-thumb{", + "width: 32px;", + "height: 32px;", + "border-radius: 6px;", + "}", ".album-art::after{", "content: \"\";", "position: absolute;", @@ -2359,8 +2368,27 @@ def _image_mime(payload: bytes) -> str: html_lines.append("
      ") html_lines.append("
      ") html_lines.append("

      Context

      ") + primary_loser = group.losers[0] if group.losers else "" + primary_disposition = ( + group.loser_disposition.get(primary_loser, "quarantine") if primary_loser else "" + ) + primary_reason = "" + if primary_loser: + for act in actions: + if act["step"] == "loser_cleanup" and act["target"] == primary_loser: + primary_reason = str(act["detail"]) + break + album_art_src = _album_art_src(group) + html_lines.append("
      ") html_lines.append("
      ") html_lines.append("
      ") + html_lines.append("
      Album art
      ") + html_lines.append("
      ") + html_lines.append("") + if album_art_src: + html_lines.append(f"") + html_lines.append("") + html_lines.append("
      ") html_lines.append("
      Kept file (winner)
      ") html_lines.append(f"
      {esc(group.winner_path)}
      ") html_lines.append("
      Group ID
      ") @@ -2371,6 +2399,31 @@ def _image_mime(payload: bytes) -> str: ) html_lines.append("
      ") html_lines.append("
      ") + html_lines.append("
      ") + html_lines.append("
      ") + html_lines.append("
      Album art
      ") + html_lines.append("
      ") + html_lines.append("") + if album_art_src: + html_lines.append(f"") + html_lines.append("") + html_lines.append("
      ") + html_lines.append("
      Loser file
      ") + if primary_loser: + html_lines.append(f"
      {esc(primary_loser)}
      ") + else: + html_lines.append("
      ") + html_lines.append("
      Disposition
      ") + if primary_disposition: + html_lines.append(f"
      {esc(primary_disposition)}
      ") + else: + html_lines.append("
      ") + if primary_reason: + html_lines.append("
      Reason
      ") + html_lines.append(f"
      {esc(primary_reason)}
      ") + html_lines.append("
      ") + html_lines.append("
      ") + html_lines.append("
      ") html_lines.append("
      ") html_lines.append("
      ") html_lines.append("

      Operations

      ") diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index 5b54a7f..d975ecc 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -1278,6 +1278,10 @@ def _run_status() -> tuple[str, str, str]: "padding: 12px;", "background: var(--card);", "}", + ".card-stack{", + "display:grid;", + "gap:10px;", + "}", ".summary{", "display:grid;", "grid-template-columns: 1fr;", @@ -1389,6 +1393,11 @@ def _run_status() -> tuple[str, str, str]: "overflow: hidden;", "background: #f7f7f7;", "}", + ".album-art.album-art-thumb{", + "width: 32px;", + "height: 32px;", + "border-radius: 6px;", + "}", ".album-art::after{", "content: \"\";", "position: absolute;", @@ -1648,8 +1657,28 @@ def _run_status() -> tuple[str, str, str]: html_lines.append("
      ") html_lines.append("
      ") html_lines.append("

      Context

      ") + loser_action = next( + (act for act in group_actions if act.step == "loser_cleanup"), None + ) + loser_path = loser_action.target if loser_action else "" + loser_disposition = "" + loser_reason = "" + if loser_action: + raw_disposition = loser_action.metadata.get("disposition") + loser_disposition = str(raw_disposition) if raw_disposition else "" + loser_reason = loser_action.detail or "" + winner_art_src = _album_art_src(grp, winner_path) + loser_art_src = _album_art_src(grp, loser_path) if loser_path else winner_art_src + html_lines.append("
      ") html_lines.append("
      ") html_lines.append("
      ") + html_lines.append("
      Album art
      ") + html_lines.append("
      ") + html_lines.append("") + if winner_art_src: + html_lines.append(f"") + html_lines.append("") + html_lines.append("
      ") html_lines.append("
      Kept file (winner)
      ") html_lines.append(f"
      {html.escape(winner_path)}
      ") html_lines.append("
      Group ID
      ") @@ -1660,6 +1689,33 @@ def _run_status() -> tuple[str, str, str]: ) html_lines.append("
      ") html_lines.append("
      ") + html_lines.append("
      ") + html_lines.append("
      ") + html_lines.append("
      Album art
      ") + html_lines.append("
      ") + html_lines.append("") + if loser_art_src: + html_lines.append(f"") + html_lines.append("") + html_lines.append("
      ") + html_lines.append("
      Loser file
      ") + if loser_path: + html_lines.append(f"
      {html.escape(loser_path)}
      ") + else: + html_lines.append("
      ") + html_lines.append("
      Disposition
      ") + if loser_disposition: + html_lines.append(f"
      {html.escape(loser_disposition)}
      ") + else: + html_lines.append("
      ") + if loser_reason: + html_lines.append("
      Reason
      ") + html_lines.append( + f"
      {html.escape(loser_reason)}
      " + ) + html_lines.append("
      ") + html_lines.append("
      ") + html_lines.append("
      ") html_lines.append("
      ") html_lines.append("
      ") html_lines.append("

      Operations

      ") From cf3e5bafa325e0763a6679b0d4ff3ee7e636c428 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Wed, 31 Dec 2025 19:03:54 -0500 Subject: [PATCH 204/606] Shrink duplicate finder album art thumbnails --- duplicate_consolidation.py | 6 +++--- duplicate_consolidation_executor.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index dc89a03..4382848 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -2109,9 +2109,9 @@ def _image_mime(payload: bytes) -> str: "background: #f7f7f7;", "}", ".album-art.album-art-thumb{", - "width: 32px;", - "height: 32px;", - "border-radius: 6px;", + "width: 24px;", + "height: 24px;", + "border-radius: 5px;", "}", ".album-art::after{", "content: \"\";", diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index d975ecc..5828a6b 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -1394,9 +1394,9 @@ def _run_status() -> tuple[str, str, str]: "background: #f7f7f7;", "}", ".album-art.album-art-thumb{", - "width: 32px;", - "height: 32px;", - "border-radius: 6px;", + "width: 24px;", + "height: 24px;", + "border-radius: 5px;", "}", ".album-art::after{", "content: \"\";", From 9bcf19cebb9c30a59fc81cc3ceefa0cc30a00f98 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Wed, 31 Dec 2025 19:09:51 -0500 Subject: [PATCH 205/606] Reduce duplicate finder album art thumbnail size --- duplicate_consolidation.py | 6 +++--- duplicate_consolidation_executor.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index 4382848..a426a54 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -2109,9 +2109,9 @@ def _image_mime(payload: bytes) -> str: "background: #f7f7f7;", "}", ".album-art.album-art-thumb{", - "width: 24px;", - "height: 24px;", - "border-radius: 5px;", + "width: 8px;", + "height: 8px;", + "border-radius: 2px;", "}", ".album-art::after{", "content: \"\";", diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index 5828a6b..db80e52 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -1394,9 +1394,9 @@ def _run_status() -> tuple[str, str, str]: "background: #f7f7f7;", "}", ".album-art.album-art-thumb{", - "width: 24px;", - "height: 24px;", - "border-radius: 5px;", + "width: 8px;", + "height: 8px;", + "border-radius: 2px;", "}", ".album-art::after{", "content: \"\";", From aa8a6919e88689a98a0cef32329546c06d28a66e Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Wed, 31 Dec 2025 19:30:20 -0500 Subject: [PATCH 206/606] Constrain duplicate finder context album art --- duplicate_consolidation.py | 3 ++- duplicate_consolidation_executor.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index 4382848..1fc6cda 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -2097,6 +2097,7 @@ def _image_mime(payload: bytes) -> str: "}", "summary.group-summary::-webkit-details-marker{ display:none; }", ".album-art{", + "display: inline-block;", "width: 42px;", "height: 42px;", "grid-column: 1;", @@ -2111,7 +2112,7 @@ def _image_mime(payload: bytes) -> str: ".album-art.album-art-thumb{", "width: 24px;", "height: 24px;", - "border-radius: 5px;", + "border-radius: 4px;", "}", ".album-art::after{", "content: \"\";", diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index 5828a6b..9ecc029 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -1382,6 +1382,7 @@ def _run_status() -> tuple[str, str, str]: "}", "summary.group-summary::-webkit-details-marker{ display:none; }", ".album-art{", + "display: inline-block;", "width: 42px;", "height: 42px;", "grid-column: 1;", @@ -1396,7 +1397,7 @@ def _run_status() -> tuple[str, str, str]: ".album-art.album-art-thumb{", "width: 24px;", "height: 24px;", - "border-radius: 5px;", + "border-radius: 4px;", "}", ".album-art::after{", "content: \"\";", From c909a2db7e0ec98134a5cc94c2bad65e52a93e4c Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Wed, 31 Dec 2025 19:32:12 -0500 Subject: [PATCH 207/606] Revert "Constrain duplicate finder context album art thumbnails (24px)" --- duplicate_consolidation.py | 1 - duplicate_consolidation_executor.py | 1 - 2 files changed, 2 deletions(-) diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index 29d51dc..a426a54 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -2097,7 +2097,6 @@ def _image_mime(payload: bytes) -> str: "}", "summary.group-summary::-webkit-details-marker{ display:none; }", ".album-art{", - "display: inline-block;", "width: 42px;", "height: 42px;", "grid-column: 1;", diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index 0340c7e..db80e52 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -1382,7 +1382,6 @@ def _run_status() -> tuple[str, str, str]: "}", "summary.group-summary::-webkit-details-marker{ display:none; }", ".album-art{", - "display: inline-block;", "width: 42px;", "height: 42px;", "grid-column: 1;", From 4537ca8816a096947cdbd6de6841e30bd0e9d029 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Wed, 31 Dec 2025 19:43:02 -0500 Subject: [PATCH 208/606] Resize duplicate finder context album art --- duplicate_consolidation.py | 7 ++++--- duplicate_consolidation_executor.py | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index a426a54..ebf0be1 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -2099,6 +2099,7 @@ def _image_mime(payload: bytes) -> str: ".album-art{", "width: 42px;", "height: 42px;", + "display: inline-block;", "grid-column: 1;", "grid-row: 1 / span 2;", "align-self: center;", @@ -2109,9 +2110,9 @@ def _image_mime(payload: bytes) -> str: "background: #f7f7f7;", "}", ".album-art.album-art-thumb{", - "width: 8px;", - "height: 8px;", - "border-radius: 2px;", + "width: 64px;", + "height: 64px;", + "border-radius: 6px;", "}", ".album-art::after{", "content: \"\";", diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index db80e52..b3727cd 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -1384,6 +1384,7 @@ def _run_status() -> tuple[str, str, str]: ".album-art{", "width: 42px;", "height: 42px;", + "display: inline-block;", "grid-column: 1;", "grid-row: 1 / span 2;", "align-self: center;", @@ -1394,9 +1395,9 @@ def _run_status() -> tuple[str, str, str]: "background: #f7f7f7;", "}", ".album-art.album-art-thumb{", - "width: 8px;", - "height: 8px;", - "border-radius: 2px;", + "width: 64px;", + "height: 64px;", + "border-radius: 6px;", "}", ".album-art::after{", "content: \"\";", From 4d867bb2947f3ac8a5bb17eacf9cb957ed4f7ecc Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Wed, 31 Dec 2025 19:47:56 -0500 Subject: [PATCH 209/606] Move context card album art to left --- duplicate_consolidation_executor.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index b3727cd..8a3e511 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -1278,6 +1278,17 @@ def _run_status() -> tuple[str, str, str]: "padding: 12px;", "background: var(--card);", "}", + ".context-card{", + "display:flex;", + "gap:12px;", + "align-items:flex-start;", + "}", + ".context-card-art{", + "flex: 0 0 auto;", + "}", + ".context-card-details{", + "flex: 1;", + "}", ".card-stack{", "display:grid;", "gap:10px;", @@ -1671,15 +1682,14 @@ def _run_status() -> tuple[str, str, str]: winner_art_src = _album_art_src(grp, winner_path) loser_art_src = _album_art_src(grp, loser_path) if loser_path else winner_art_src html_lines.append("
      ") - html_lines.append("
      ") - html_lines.append("
      ") - html_lines.append("
      Album art
      ") - html_lines.append("
      ") + html_lines.append("
      ") + html_lines.append("
      ") html_lines.append("") if winner_art_src: html_lines.append(f"") html_lines.append("") html_lines.append("
      ") + html_lines.append("
      ") html_lines.append("
      Kept file (winner)
      ") html_lines.append(f"
      {html.escape(winner_path)}
      ") html_lines.append("
      Group ID
      ") @@ -1690,15 +1700,14 @@ def _run_status() -> tuple[str, str, str]: ) html_lines.append("
      ") html_lines.append("
      ") - html_lines.append("
      ") - html_lines.append("
      ") - html_lines.append("
      Album art
      ") - html_lines.append("
      ") + html_lines.append("
      ") + html_lines.append("
      ") html_lines.append("") if loser_art_src: html_lines.append(f"") html_lines.append("") html_lines.append("
      ") + html_lines.append("
      ") html_lines.append("
      Loser file
      ") if loser_path: html_lines.append(f"
      {html.escape(loser_path)}
      ") From 99fcf9d4111b90356802317f0cdab01c12c39716 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Wed, 31 Dec 2025 20:15:05 -0500 Subject: [PATCH 210/606] Align context card artwork in preview --- duplicate_consolidation.py | 55 ++++++++++++++++++++--------- duplicate_consolidation_executor.py | 43 +++++++++++++++------- 2 files changed, 69 insertions(+), 29 deletions(-) diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index ebf0be1..c884f53 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -1898,20 +1898,29 @@ def _group_disposition_count(group: GroupPlan) -> int: if group.loser_disposition.get(loser, "quarantine") != "retain" ) - def _album_art_src(group: GroupPlan) -> str | None: + def _album_art_src( + group: GroupPlan, + track_path: str, + include_group_chosen: bool = True, + ) -> str | None: chosen_hash = "" - if isinstance(group.chosen_artwork_source, Mapping): + if include_group_chosen and isinstance(group.chosen_artwork_source, Mapping): raw_hash = group.chosen_artwork_source.get("hash") if raw_hash: chosen_hash = str(raw_hash) fallback: ArtworkCandidate | None = None + path_fallback: ArtworkCandidate | None = None for candidate in group.artwork_candidates: if not candidate.bytes: continue if chosen_hash and candidate.hash == chosen_hash: return _image_data_uri(candidate.bytes) + if track_path and candidate.path == track_path and path_fallback is None: + path_fallback = candidate if fallback is None: fallback = candidate + if path_fallback and path_fallback.bytes: + return _image_data_uri(path_fallback.bytes) if fallback and fallback.bytes: return _image_data_uri(fallback.bytes) return None @@ -1993,6 +2002,17 @@ def _image_mime(payload: bytes) -> str: "padding: 12px;", "background: var(--card);", "}", + ".context-card{", + "display:flex;", + "gap:12px;", + "align-items:flex-start;", + "}", + ".context-card-art{", + "flex: 0 0 auto;", + "}", + ".context-card-details{", + "flex: 1;", + "}", ".card-stack{", "display:grid;", "gap:10px;", @@ -2341,7 +2361,7 @@ def _image_mime(payload: bytes) -> str: f"data-search='{esc(search_text.lower())}' " f"data-quarantine-count='{group_disposition_count}'>" ) - album_art_src = _album_art_src(group) + album_art_src = _album_art_src(group, group.winner_path) html_lines.append("") html_lines.append("") if album_art_src: @@ -2379,17 +2399,21 @@ def _image_mime(payload: bytes) -> str: if act["step"] == "loser_cleanup" and act["target"] == primary_loser: primary_reason = str(act["detail"]) break - album_art_src = _album_art_src(group) + winner_art_src = _album_art_src(group, group.winner_path) + loser_art_src = ( + _album_art_src(group, primary_loser, include_group_chosen=False) + if primary_loser + else None + ) html_lines.append("
      ") - html_lines.append("
      ") - html_lines.append("
      ") - html_lines.append("
      Album art
      ") - html_lines.append("
      ") + html_lines.append("
      ") + html_lines.append("
      ") html_lines.append("") - if album_art_src: - html_lines.append(f"") + if winner_art_src: + html_lines.append(f"") html_lines.append("") html_lines.append("
      ") + html_lines.append("
      ") html_lines.append("
      Kept file (winner)
      ") html_lines.append(f"
      {esc(group.winner_path)}
      ") html_lines.append("
      Group ID
      ") @@ -2400,15 +2424,14 @@ def _image_mime(payload: bytes) -> str: ) html_lines.append("
      ") html_lines.append("
      ") - html_lines.append("
      ") - html_lines.append("
      ") - html_lines.append("
      Album art
      ") - html_lines.append("
      ") + html_lines.append("
      ") + html_lines.append("
      ") html_lines.append("") - if album_art_src: - html_lines.append(f"") + if loser_art_src: + html_lines.append(f"") html_lines.append("") html_lines.append("
      ") + html_lines.append("
      ") html_lines.append("
      Loser file
      ") if primary_loser: html_lines.append(f"
      {esc(primary_loser)}
      ") diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index b3727cd..1014d6f 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -1126,16 +1126,20 @@ def _action_badges(group_actions: List[ExecutionAction]) -> List[str]: badges.append(f"artwork copied ({art_success})") return badges - def _album_art_src(group: object, winner_path: str) -> str | None: + def _album_art_src( + group: object, + track_path: str, + include_group_chosen: bool = True, + ) -> str | None: candidates: List[str] = [] - if group is not None: + if include_group_chosen and group is not None: chosen = getattr(group, "chosen_artwork_source", None) if isinstance(chosen, Mapping): chosen_path = chosen.get("path") if isinstance(chosen_path, str) and chosen_path: candidates.append(chosen_path) - if winner_path: - candidates.append(winner_path) + if track_path: + candidates.append(track_path) for path in candidates: payload = _extract_artwork_bytes(path) if payload: @@ -1278,6 +1282,17 @@ def _run_status() -> tuple[str, str, str]: "padding: 12px;", "background: var(--card);", "}", + ".context-card{", + "display:flex;", + "gap:12px;", + "align-items:flex-start;", + "}", + ".context-card-art{", + "flex: 0 0 auto;", + "}", + ".context-card-details{", + "flex: 1;", + "}", ".card-stack{", "display:grid;", "gap:10px;", @@ -1669,17 +1684,20 @@ def _run_status() -> tuple[str, str, str]: loser_disposition = str(raw_disposition) if raw_disposition else "" loser_reason = loser_action.detail or "" winner_art_src = _album_art_src(grp, winner_path) - loser_art_src = _album_art_src(grp, loser_path) if loser_path else winner_art_src + loser_art_src = ( + _album_art_src(grp, loser_path, include_group_chosen=False) + if loser_path + else None + ) html_lines.append("
      ") - html_lines.append("
      ") - html_lines.append("
      ") - html_lines.append("
      Album art
      ") - html_lines.append("
      ") + html_lines.append("
      ") + html_lines.append("
      ") html_lines.append("") if winner_art_src: html_lines.append(f"") html_lines.append("") html_lines.append("
      ") + html_lines.append("
      ") html_lines.append("
      Kept file (winner)
      ") html_lines.append(f"
      {html.escape(winner_path)}
      ") html_lines.append("
      Group ID
      ") @@ -1690,15 +1708,14 @@ def _run_status() -> tuple[str, str, str]: ) html_lines.append("
      ") html_lines.append("
      ") - html_lines.append("
      ") - html_lines.append("
      ") - html_lines.append("
      Album art
      ") - html_lines.append("
      ") + html_lines.append("
      ") + html_lines.append("
      ") html_lines.append("") if loser_art_src: html_lines.append(f"") html_lines.append("") html_lines.append("
      ") + html_lines.append("
      ") html_lines.append("
      Loser file
      ") if loser_path: html_lines.append(f"
      {html.escape(loser_path)}
      ") From 8fedb0b1ff134423572eb1164deaf554b0d7ff05 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Wed, 31 Dec 2025 20:15:17 -0500 Subject: [PATCH 211/606] Revert "Move context card album art to left" --- duplicate_consolidation_executor.py | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index 8a3e511..b3727cd 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -1278,17 +1278,6 @@ def _run_status() -> tuple[str, str, str]: "padding: 12px;", "background: var(--card);", "}", - ".context-card{", - "display:flex;", - "gap:12px;", - "align-items:flex-start;", - "}", - ".context-card-art{", - "flex: 0 0 auto;", - "}", - ".context-card-details{", - "flex: 1;", - "}", ".card-stack{", "display:grid;", "gap:10px;", @@ -1682,14 +1671,15 @@ def _run_status() -> tuple[str, str, str]: winner_art_src = _album_art_src(grp, winner_path) loser_art_src = _album_art_src(grp, loser_path) if loser_path else winner_art_src html_lines.append("
      ") - html_lines.append("
      ") - html_lines.append("
      ") + html_lines.append("
      ") + html_lines.append("
      ") + html_lines.append("
      Album art
      ") + html_lines.append("
      ") html_lines.append("") if winner_art_src: html_lines.append(f"") html_lines.append("") html_lines.append("
      ") - html_lines.append("
      ") html_lines.append("
      Kept file (winner)
      ") html_lines.append(f"
      {html.escape(winner_path)}
      ") html_lines.append("
      Group ID
      ") @@ -1700,14 +1690,15 @@ def _run_status() -> tuple[str, str, str]: ) html_lines.append("
      ") html_lines.append("
      ") - html_lines.append("
      ") - html_lines.append("
      ") + html_lines.append("
      ") + html_lines.append("
      ") + html_lines.append("
      Album art
      ") + html_lines.append("
      ") html_lines.append("") if loser_art_src: html_lines.append(f"") html_lines.append("") html_lines.append("
      ") - html_lines.append("
      ") html_lines.append("
      Loser file
      ") if loser_path: html_lines.append(f"
      {html.escape(loser_path)}
      ") From 2e80e3aacea8576f1385c108934647dde8a38e7a Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Wed, 31 Dec 2025 20:26:24 -0500 Subject: [PATCH 212/606] Add multi-loser context cards to duplicate report --- duplicate_consolidation.py | 77 +++++++++++++------------- duplicate_consolidation_executor.py | 83 ++++++++++++++++------------- 2 files changed, 84 insertions(+), 76 deletions(-) diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index c884f53..30f0bad 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -2389,22 +2389,13 @@ def _image_mime(payload: bytes) -> str: html_lines.append("
      ") html_lines.append("
      ") html_lines.append("

      Context

      ") - primary_loser = group.losers[0] if group.losers else "" - primary_disposition = ( - group.loser_disposition.get(primary_loser, "quarantine") if primary_loser else "" - ) - primary_reason = "" - if primary_loser: - for act in actions: - if act["step"] == "loser_cleanup" and act["target"] == primary_loser: - primary_reason = str(act["detail"]) - break + loser_reason_map = { + act["target"]: str(act["detail"]) + for act in actions + if act["step"] == "loser_cleanup" + } + loser_paths = group.losers or [""] winner_art_src = _album_art_src(group, group.winner_path) - loser_art_src = ( - _album_art_src(group, primary_loser, include_group_chosen=False) - if primary_loser - else None - ) html_lines.append("
      ") html_lines.append("
      ") html_lines.append("
      ") @@ -2424,29 +2415,39 @@ def _image_mime(payload: bytes) -> str: ) html_lines.append("
      ") html_lines.append("
      ") - html_lines.append("
      ") - html_lines.append("
      ") - html_lines.append("") - if loser_art_src: - html_lines.append(f"") - html_lines.append("") - html_lines.append("
      ") - html_lines.append("
      ") - html_lines.append("
      Loser file
      ") - if primary_loser: - html_lines.append(f"
      {esc(primary_loser)}
      ") - else: - html_lines.append("
      ") - html_lines.append("
      Disposition
      ") - if primary_disposition: - html_lines.append(f"
      {esc(primary_disposition)}
      ") - else: - html_lines.append("
      ") - if primary_reason: - html_lines.append("
      Reason
      ") - html_lines.append(f"
      {esc(primary_reason)}
      ") - html_lines.append("
      ") - html_lines.append("
      ") + for loser_path in loser_paths: + loser_disposition = ( + group.loser_disposition.get(loser_path, "quarantine") if loser_path else "" + ) + loser_reason = loser_reason_map.get(loser_path, "") + loser_art_src = ( + _album_art_src(group, loser_path, include_group_chosen=False) + if loser_path + else None + ) + html_lines.append("
      ") + html_lines.append("
      ") + html_lines.append("") + if loser_art_src: + html_lines.append(f"") + html_lines.append("") + html_lines.append("
      ") + html_lines.append("
      ") + html_lines.append("
      Loser file
      ") + if loser_path: + html_lines.append(f"
      {esc(loser_path)}
      ") + else: + html_lines.append("
      ") + html_lines.append("
      Disposition
      ") + if loser_disposition: + html_lines.append(f"
      {esc(loser_disposition)}
      ") + else: + html_lines.append("
      ") + if loser_reason: + html_lines.append("
      Reason
      ") + html_lines.append(f"
      {esc(loser_reason)}
      ") + html_lines.append("
      ") + html_lines.append("
      ") html_lines.append("
      ") html_lines.append("
      ") html_lines.append("
      ") diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index 1014d6f..29552f5 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -1673,22 +1673,8 @@ def _run_status() -> tuple[str, str, str]: html_lines.append("
      ") html_lines.append("
      ") html_lines.append("

      Context

      ") - loser_action = next( - (act for act in group_actions if act.step == "loser_cleanup"), None - ) - loser_path = loser_action.target if loser_action else "" - loser_disposition = "" - loser_reason = "" - if loser_action: - raw_disposition = loser_action.metadata.get("disposition") - loser_disposition = str(raw_disposition) if raw_disposition else "" - loser_reason = loser_action.detail or "" + cleanup_actions = [act for act in group_actions if act.step == "loser_cleanup"] winner_art_src = _album_art_src(grp, winner_path) - loser_art_src = ( - _album_art_src(grp, loser_path, include_group_chosen=False) - if loser_path - else None - ) html_lines.append("
      ") html_lines.append("
      ") html_lines.append("
      ") @@ -1708,31 +1694,52 @@ def _run_status() -> tuple[str, str, str]: ) html_lines.append("
      ") html_lines.append("
      ") - html_lines.append("
      ") - html_lines.append("
      ") - html_lines.append("") - if loser_art_src: - html_lines.append(f"") - html_lines.append("") - html_lines.append("
      ") - html_lines.append("
      ") - html_lines.append("
      Loser file
      ") - if loser_path: - html_lines.append(f"
      {html.escape(loser_path)}
      ") - else: - html_lines.append("
      ") - html_lines.append("
      Disposition
      ") - if loser_disposition: - html_lines.append(f"
      {html.escape(loser_disposition)}
      ") + if cleanup_actions: + loser_entries = cleanup_actions else: - html_lines.append("
      ") - if loser_reason: - html_lines.append("
      Reason
      ") - html_lines.append( - f"
      {html.escape(loser_reason)}
      " + loser_entries = [None] + for loser_action in loser_entries: + loser_path = loser_action.target if loser_action else "" + loser_disposition = "" + loser_reason = "" + if loser_action: + raw_disposition = loser_action.metadata.get("disposition") + loser_disposition = str(raw_disposition) if raw_disposition else "" + loser_reason = loser_action.detail or "" + loser_art_src = ( + _album_art_src(grp, loser_path, include_group_chosen=False) + if loser_path + else None ) - html_lines.append("
      ") - html_lines.append("
      ") + html_lines.append("
      ") + html_lines.append("
      ") + html_lines.append("") + if loser_art_src: + html_lines.append(f"") + html_lines.append("") + html_lines.append("
      ") + html_lines.append("
      ") + html_lines.append("
      Loser file
      ") + if loser_path: + html_lines.append( + f"
      {html.escape(loser_path)}
      " + ) + else: + html_lines.append("
      ") + html_lines.append("
      Disposition
      ") + if loser_disposition: + html_lines.append( + f"
      {html.escape(loser_disposition)}
      " + ) + else: + html_lines.append("
      ") + if loser_reason: + html_lines.append("
      Reason
      ") + html_lines.append( + f"
      {html.escape(loser_reason)}
      " + ) + html_lines.append("
      ") + html_lines.append("
      ") html_lines.append("
      ") html_lines.append("
      ") html_lines.append("
      ") From 9f510cc48cbb18ada3c55f161b20092d481e8747 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Thu, 1 Jan 2026 10:37:12 -0500 Subject: [PATCH 213/606] Gate consolidation on artwork similarity --- duplicate_consolidation.py | 656 +++++++++++++++++++++---------------- 1 file changed, 374 insertions(+), 282 deletions(-) diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index 30f0bad..19e5c23 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -47,6 +47,8 @@ LOSSLESS_EXTS = {".flac", ".wav", ".alac", ".ape", ".aiff", ".aif"} EXACT_DUPLICATE_THRESHOLD = 0.02 NEAR_DUPLICATE_THRESHOLD = 0.10 +ARTWORK_HASH_SIZE = 8 +ARTWORK_SIMILARITY_THRESHOLD = 10 DEFAULT_MAX_CANDIDATES = 5000 DEFAULT_MAX_COMPARISONS = 50_000 DEFAULT_TIMEOUT_SEC = 15.0 @@ -303,6 +305,80 @@ def _extract_image_dimensions(payload: bytes) -> tuple[Optional[int], Optional[i return None, None +def _image_resample_filter(): + if not Image: + return None + resampling = getattr(Image, "Resampling", Image) + return resampling.LANCZOS + + +def _artwork_perceptual_hash(payload: bytes) -> int | None: + if not payload or not Image: + return None + try: + with Image.open(io.BytesIO(payload)) as img: + resized = img.convert("L").resize( + (ARTWORK_HASH_SIZE, ARTWORK_HASH_SIZE), + resample=_image_resample_filter(), + ) + pixels = list(resized.getdata()) + except Exception: + return None + if not pixels: + return None + avg = sum(pixels) / len(pixels) + value = 0 + for px in pixels: + value = (value << 1) | (1 if px >= avg else 0) + return value + + +def _hamming_distance(left: int, right: int) -> int: + xor = left ^ right + return xor.bit_count() if hasattr(xor, "bit_count") else bin(xor).count("1") + + +def _artwork_hash_for_track(track: "DuplicateTrack") -> int | None: + blob = _best_artwork_blob(track.artwork) + if not blob or not blob.bytes: + return None + return _artwork_perceptual_hash(blob.bytes) + + +def _artwork_tracks_similar( + left: "DuplicateTrack", + right: "DuplicateTrack", + hashes: Mapping[str, int | None], +) -> bool: + left_hash = hashes.get(left.path) + right_hash = hashes.get(right.path) + if left_hash is None or right_hash is None: + return True + return _hamming_distance(left_hash, right_hash) <= ARTWORK_SIMILARITY_THRESHOLD + + +def _split_by_artwork_similarity( + tracks: Sequence["DuplicateTrack"], +) -> tuple[List[List["DuplicateTrack"]], Dict[str, int | None]]: + if len(tracks) <= 1: + return [list(tracks)], {} + hashes: Dict[str, int | None] = {t.path: _artwork_hash_for_track(t) for t in tracks} + pending = sorted(tracks, key=lambda t: t.path) + groups: List[List[DuplicateTrack]] = [] + while pending: + anchor = pending.pop(0) + group = [anchor] + remaining: List[DuplicateTrack] = [] + for candidate in pending: + if any(_artwork_tracks_similar(candidate, member, hashes) for member in group): + group.append(candidate) + else: + remaining.append(candidate) + pending = remaining + groups.append(group) + return groups, hashes + + def _read_audio_file(path: str): if MutagenFile is None: return None, "mutagen unavailable" @@ -1459,306 +1535,322 @@ def build_consolidation_plan( plan_placeholders = False for cluster in sorted(clusters, key=lambda c: _stable_group_id([t.path for t in c.tracks])): cluster_tracks = cluster.tracks - contexts: Dict[str, str] = {} - context_evidence: Dict[str, List[str]] = {} - release_sizes: Dict[str, str] = {} - release_size_evidence: Dict[str, str] = {} - for t in cluster_tracks: - ctx, evidence = _classify_context(t) - contexts[t.path] = ctx - t.context_evidence = evidence - context_evidence[t.path] = evidence - tags = t.current_tags if isinstance(t.current_tags, Mapping) else t.tags - tags = tags or {} - release_size, release_reason = _release_size_context(tags) - release_sizes[t.path] = release_size - release_size_evidence[t.path] = release_reason - - album_label_by_key: Dict[tuple[str, str], str] = {} - album_keys: set[tuple[str, str]] = set() - for t in cluster_tracks: - tags = t.current_tags if isinstance(t.current_tags, Mapping) else t.tags - tags = tags or {} - album_key = _normalize_album_key(tags, t.path) - if album_key: - album_keys.add(album_key) - if album_key not in album_label_by_key: - album_label_by_key[album_key] = _album_display_label(tags) - - release_notes: List[str] = [] - if len(album_keys) > 1: - album_labels = [album_label_by_key.get(key, "Unknown album") for key in sorted(album_keys)] - album_summary = ", ".join(sorted({label for label in album_labels if label})) - cross_album_note = "Identical recording across albums; consolidated into one canonical track." - if album_summary: - cross_album_note = f"{cross_album_note} Albums: {album_summary}." - release_notes.append(cross_album_note) - if len({contexts[path] for path in contexts}) > 1: - release_notes.append("Cluster spans multiple release contexts; using one canonical winner across variants.") - - group_tracks = cluster_tracks - group_paths = {t.path for t in group_tracks} - group_contexts = {path: contexts[path] for path in group_paths} - group_context_evidence = {path: context_evidence[path] for path in group_paths} - group_decisions = [ - d for d in cluster.decisions if d.anchor_path in group_paths and d.candidate_path in group_paths - ] - - quality_sorted = sorted(group_tracks, key=lambda t: _quality_tuple(t, group_contexts[t.path]), reverse=True) - winner = quality_sorted[0] - losers = [t.path for t in quality_sorted[1:]] - runner_up = quality_sorted[1] if len(quality_sorted) > 1 else None - - metadata_source = _select_metadata_source(group_tracks, group_contexts) - planned_tags, tag_source, tag_source_reason, tag_source_evidence = _build_planned_tags( - winner, metadata_source, group_contexts - ) - group_review: List[str] = list(release_notes) - if not metadata_source: - review_flags.append(f"Missing metadata source for group containing {winner.path}.") - group_review.append("Metadata source missing; tags may be incomplete.") - - meta_changes = _metadata_changes(winner, planned_tags) - winner_context = group_contexts.get(winner.path, "unknown") - winner_quality = _quality_rationale(winner, runner_up, winner_context) - - distance_samples: List[float] = [] - pair_distances: Dict[str, Dict[str, float]] = {t.path: {} for t in group_tracks} - missing_fingerprints = any(not t.fingerprint for t in group_tracks) - for idx, t in enumerate(group_tracks): - for other in group_tracks[idx + 1 :]: - dist = fingerprint_distance(t.fingerprint, other.fingerprint) - distance_samples.append(dist) - pair_distances[t.path][other.path] = dist - pair_distances[other.path][t.path] = dist - max_distance = max(distance_samples) if distance_samples else (1.0 if missing_fingerprints else 0.0) - - track_quality: Dict[str, Dict[str, object]] = { - t.path: { - "container": t.container or t.ext.replace(".", "").upper(), - "codec": t.codec, - "bitrate": t.bitrate, - "sample_rate": t.sample_rate, - "bit_depth": t.bit_depth, - "channels": t.channels, - "is_lossless": t.is_lossless, + artwork_groups, artwork_hashes = _split_by_artwork_similarity(cluster_tracks) + artwork_split = len(artwork_groups) > 1 + for group_tracks in sorted(artwork_groups, key=lambda g: _stable_group_id([t.path for t in g])): + contexts: Dict[str, str] = {} + context_evidence: Dict[str, List[str]] = {} + release_sizes: Dict[str, str] = {} + release_size_evidence: Dict[str, str] = {} + for t in group_tracks: + ctx, evidence = _classify_context(t) + contexts[t.path] = ctx + t.context_evidence = evidence + context_evidence[t.path] = evidence + tags = t.current_tags if isinstance(t.current_tags, Mapping) else t.tags + tags = tags or {} + release_size, release_reason = _release_size_context(tags) + release_sizes[t.path] = release_size + release_size_evidence[t.path] = release_reason + + album_label_by_key: Dict[tuple[str, str], str] = {} + album_keys: set[tuple[str, str]] = set() + for t in group_tracks: + tags = t.current_tags if isinstance(t.current_tags, Mapping) else t.tags + tags = tags or {} + album_key = _normalize_album_key(tags, t.path) + if album_key: + album_keys.add(album_key) + if album_key not in album_label_by_key: + album_label_by_key[album_key] = _album_display_label(tags) + + suppress_review_flags = artwork_split and len(group_tracks) == 1 + release_notes: List[str] = [] + if len(album_keys) > 1 and not suppress_review_flags: + album_labels = [album_label_by_key.get(key, "Unknown album") for key in sorted(album_keys)] + album_summary = ", ".join(sorted({label for label in album_labels if label})) + cross_album_note = "Identical recording across albums; consolidated into one canonical track." + if album_summary: + cross_album_note = f"{cross_album_note} Albums: {album_summary}." + release_notes.append(cross_album_note) + if len({contexts[path] for path in contexts}) > 1 and not suppress_review_flags: + release_notes.append("Cluster spans multiple release contexts; using one canonical winner across variants.") + + group_paths = {t.path for t in group_tracks} + group_contexts = {path: contexts[path] for path in group_paths} + group_context_evidence = {path: context_evidence[path] for path in group_paths} + group_decisions = [ + d for d in cluster.decisions if d.anchor_path in group_paths and d.candidate_path in group_paths + ] + + quality_sorted = sorted(group_tracks, key=lambda t: _quality_tuple(t, group_contexts[t.path]), reverse=True) + winner = quality_sorted[0] + losers = [t.path for t in quality_sorted[1:]] + runner_up = quality_sorted[1] if len(quality_sorted) > 1 else None + + metadata_source = _select_metadata_source(group_tracks, group_contexts) + planned_tags, tag_source, tag_source_reason, tag_source_evidence = _build_planned_tags( + winner, metadata_source, group_contexts + ) + group_review: List[str] = list(release_notes) + if not metadata_source and not suppress_review_flags: + review_flags.append(f"Missing metadata source for group containing {winner.path}.") + group_review.append("Metadata source missing; tags may be incomplete.") + + meta_changes = _metadata_changes(winner, planned_tags) + winner_context = group_contexts.get(winner.path, "unknown") + winner_quality = _quality_rationale(winner, runner_up, winner_context) + + distance_samples: List[float] = [] + pair_distances: Dict[str, Dict[str, float]] = {t.path: {} for t in group_tracks} + missing_fingerprints = any(not t.fingerprint for t in group_tracks) + for idx, t in enumerate(group_tracks): + for other in group_tracks[idx + 1 :]: + dist = fingerprint_distance(t.fingerprint, other.fingerprint) + distance_samples.append(dist) + pair_distances[t.path][other.path] = dist + pair_distances[other.path][t.path] = dist + max_distance = max(distance_samples) if distance_samples else (1.0 if missing_fingerprints else 0.0) + + track_quality: Dict[str, Dict[str, object]] = { + t.path: { + "container": t.container or t.ext.replace(".", "").upper(), + "codec": t.codec, + "bitrate": t.bitrate, + "sample_rate": t.sample_rate, + "bit_depth": t.bit_depth, + "channels": t.channels, + "is_lossless": t.is_lossless, + } + for t in quality_sorted } - for t in quality_sorted - } - single_art_track, single_art_blob, ambiguous_single_art = _select_single_release_artwork_candidate( - group_tracks, release_sizes - ) - overall_art_track, overall_art_blob, ambiguous_overall_art = _select_overall_artwork_candidate( - group_tracks - ) - artwork_actions: List[ArtworkDirective] = [] - chosen_artwork_source: Dict[str, object] = {"path": None, "reason": ""} - artwork_status = "unchanged" - artwork_evidence: List[str] = [] - winner_release = release_sizes.get(winner.path, "unknown") - winner_multi_release = winner_release == "multi" - if winner_release != "unknown": - artwork_evidence.append( - f"Winner release size classified as {winner_release} ({release_size_evidence.get(winner.path)})." + single_art_track, single_art_blob, ambiguous_single_art = _select_single_release_artwork_candidate( + group_tracks, release_sizes ) - - if single_art_blob: - chosen_artwork_source = { - "path": single_art_track.path if single_art_track else None, - "hash": single_art_blob.hash, - "size": single_art_blob.size, - "width": single_art_blob.width, - "height": single_art_blob.height, - "context": "single-release", - "reason": "Best single-release artwork by resolution/size", - } - artwork_evidence.append( - f"Single-release artwork selected from {single_art_track.path if single_art_track else 'unknown'} " - f"({single_art_blob.width}x{single_art_blob.height}, {single_art_blob.size} bytes)" + overall_art_track, overall_art_blob, ambiguous_overall_art = _select_overall_artwork_candidate( + group_tracks ) - if single_art_track: + artwork_actions: List[ArtworkDirective] = [] + chosen_artwork_source: Dict[str, object] = {"path": None, "reason": ""} + artwork_status = "unchanged" + artwork_evidence: List[str] = [] + winner_release = release_sizes.get(winner.path, "unknown") + winner_multi_release = winner_release == "multi" + if winner_release != "unknown": artwork_evidence.append( - f"Single-release classification: {release_size_evidence.get(single_art_track.path)}." + f"Winner release size classified as {winner_release} ({release_size_evidence.get(winner.path)})." ) - elif overall_art_blob: - chosen_artwork_source = { - "path": overall_art_track.path if overall_art_track else None, - "hash": overall_art_blob.hash, - "size": overall_art_blob.size, - "width": overall_art_blob.width, - "height": overall_art_blob.height, - "context": "fallback", - "reason": "No single artwork found; best available artwork", - } - artwork_evidence.append( - f"No single artwork found; using best available artwork from {overall_art_track.path if overall_art_track else 'unknown'} " - f"({overall_art_blob.width}x{overall_art_blob.height}, {overall_art_blob.size} bytes)" - ) - else: - chosen_artwork_source["reason"] = "No artwork candidates available" - artwork_status = "none found" - artwork_evidence.append("No readable artwork candidates were discovered.") - - if winner_multi_release and single_art_blob and single_art_track: - if not winner.cover_hash or winner.cover_hash != single_art_blob.hash: - artwork_status = "apply" - artwork_actions.append( - ArtworkDirective( - source=single_art_track.path, - target=winner.path, - reason="Migrate single-release artwork onto multi-track album winner", + if artwork_split: + if len(group_tracks) == 1: + artwork_evidence.append( + "Artwork differs from other audio duplicates; preserving this track as an intentional variant." ) - ) - artwork_evidence.append( - "Multi-track album winner will receive single-release artwork to preserve unique single cover." - ) - else: - artwork_status = "unchanged" - chosen_artwork_source["reason"] = "Winner already has selected single-release artwork" - artwork_evidence.append("Winner already matches selected single-release artwork hash; no copy needed.") - elif winner_multi_release and not single_art_blob and overall_art_blob and overall_art_track: - source_release = release_sizes.get(overall_art_track.path, "unknown") - if source_release == "multi" and overall_art_track.path != winner.path: - artwork_status = "unchanged" - chosen_artwork_source["reason"] = "Skipped multi-track artwork migration to avoid cross-album contamination" + else: + artwork_evidence.append( + "Artwork similarity gate grouped only tracks with matching covers for consolidation." + ) + + if single_art_blob: + chosen_artwork_source = { + "path": single_art_track.path if single_art_track else None, + "hash": single_art_blob.hash, + "size": single_art_blob.size, + "width": single_art_blob.width, + "height": single_art_blob.height, + "context": "single-release", + "reason": "Best single-release artwork by resolution/size", + } artwork_evidence.append( - "Skipping artwork migration between multi-track releases to avoid cross-album contamination." + f"Single-release artwork selected from {single_art_track.path if single_art_track else 'unknown'} " + f"({single_art_blob.width}x{single_art_blob.height}, {single_art_blob.size} bytes)" ) - elif not winner.cover_hash or winner.cover_hash != overall_art_blob.hash: - artwork_status = "apply" - artwork_actions.append( - ArtworkDirective( - source=overall_art_track.path, - target=winner.path, - reason="Fill missing artwork from best available source", + if single_art_track: + artwork_evidence.append( + f"Single-release classification: {release_size_evidence.get(single_art_track.path)}." ) - ) + elif overall_art_blob: + chosen_artwork_source = { + "path": overall_art_track.path if overall_art_track else None, + "hash": overall_art_blob.hash, + "size": overall_art_blob.size, + "width": overall_art_blob.width, + "height": overall_art_blob.height, + "context": "fallback", + "reason": "No single artwork found; best available artwork", + } artwork_evidence.append( - "No single-release artwork available; applying best available artwork to album winner." + f"No single artwork found; using best available artwork from {overall_art_track.path if overall_art_track else 'unknown'} " + f"({overall_art_blob.width}x{overall_art_blob.height}, {overall_art_blob.size} bytes)" ) else: - artwork_evidence.append("Album winner artwork already matches best available source.") - else: - if winner_context == "single": - artwork_evidence.append("Single-context winner retains its own artwork; album art will not be pulled.") - chosen_artwork_source["reason"] = ( - chosen_artwork_source.get("reason") or "Single winner keeps artwork" + chosen_artwork_source["reason"] = "No artwork candidates available" + artwork_status = "none found" + artwork_evidence.append("No readable artwork candidates were discovered.") + + if winner_multi_release and single_art_blob and single_art_track: + if not winner.cover_hash or winner.cover_hash != single_art_blob.hash: + artwork_status = "apply" + artwork_actions.append( + ArtworkDirective( + source=single_art_track.path, + target=winner.path, + reason="Migrate single-release artwork onto multi-track album winner", + ) + ) + artwork_evidence.append( + "Multi-track album winner will receive single-release artwork to preserve unique single cover." + ) + else: + artwork_status = "unchanged" + chosen_artwork_source["reason"] = "Winner already has selected single-release artwork" + artwork_evidence.append("Winner already matches selected single-release artwork hash; no copy needed.") + elif winner_multi_release and not single_art_blob and overall_art_blob and overall_art_track: + source_release = release_sizes.get(overall_art_track.path, "unknown") + if source_release == "multi" and overall_art_track.path != winner.path: + artwork_status = "unchanged" + chosen_artwork_source["reason"] = "Skipped multi-track artwork migration to avoid cross-album contamination" + artwork_evidence.append( + "Skipping artwork migration between multi-track releases to avoid cross-album contamination." + ) + elif not winner.cover_hash or winner.cover_hash != overall_art_blob.hash: + artwork_status = "apply" + artwork_actions.append( + ArtworkDirective( + source=overall_art_track.path, + target=winner.path, + reason="Fill missing artwork from best available source", + ) + ) + artwork_evidence.append( + "No single-release artwork available; applying best available artwork to album winner." + ) + else: + artwork_evidence.append("Album winner artwork already matches best available source.") + else: + if winner_context == "single": + artwork_evidence.append("Single-context winner retains its own artwork; album art will not be pulled.") + chosen_artwork_source["reason"] = ( + chosen_artwork_source.get("reason") or "Single winner keeps artwork" + ) + elif not single_art_blob and not overall_art_blob and not suppress_review_flags: + group_review.append("No artwork available to apply.") + + cover_hashes = {t.cover_hash for t in group_tracks if t.cover_hash} + ambiguous_art = ambiguous_single_art or ambiguous_overall_art + if len(cover_hashes) > 1: + ambiguous_art = True + artwork_evidence.append("Conflicting embedded artwork hashes between candidates.") + if ambiguous_art and not suppress_review_flags: + review_flags.append( + f"Artwork selection ambiguous for group {_stable_group_id([t.path for t in group_tracks])}." ) - elif not single_art_blob and not overall_art_blob: - group_review.append("No artwork available to apply.") - - cover_hashes = {t.cover_hash for t in group_tracks if t.cover_hash} - ambiguous_art = ambiguous_single_art or ambiguous_overall_art - if len(cover_hashes) > 1: - ambiguous_art = True - artwork_evidence.append("Conflicting embedded artwork hashes between candidates.") - if ambiguous_art: - review_flags.append( - f"Artwork selection ambiguous for group {_stable_group_id([t.path for t in group_tracks])}." - ) - if not (single_art_blob or overall_art_blob): - missing_reason = "No artwork candidates available" - if any(t.artwork_error for t in group_tracks): - missing_reason = "Artwork present but unreadable" - review_flags.append(f"{missing_reason} for group {_stable_group_id([t.path for t in group_tracks])}.") - chosen_artwork_source["reason"] = missing_reason - artwork_status = "none found" - group_review.append(missing_reason) - - required_tag_gaps = [key for key in ("artist", "title") if not planned_tags.get(key)] - if required_tag_gaps: - group_review.append(f"Missing critical tags: {', '.join(sorted(required_tag_gaps))}.") - - group_confidence = "High (identical fingerprint cluster)" - if missing_fingerprints: - group_confidence = "Low (missing fingerprints in cluster)" - group_review.append("One or more tracks missing fingerprints; requires review.") - near_distances = [d for d in distance_samples if d > exact_duplicate_threshold] - if near_distances: - group_confidence = f"Medium (near-duplicate fingerprint distance up to {max_distance:.3f})" - group_review.append( - f"Audio match requires review (max distance {max_distance:.3f}; near-duplicate threshold {near_duplicate_threshold:.3f})." - ) - if max_distance > near_duplicate_threshold: - group_confidence = f"Low (max fingerprint distance {max_distance:.3f})" - if not missing_fingerprints: + if not (single_art_blob or overall_art_blob) and not suppress_review_flags: + missing_reason = "No artwork candidates available" + if any(t.artwork_error for t in group_tracks): + missing_reason = "Artwork present but unreadable" + review_flags.append(f"{missing_reason} for group {_stable_group_id([t.path for t in group_tracks])}.") + chosen_artwork_source["reason"] = missing_reason + artwork_status = "none found" + group_review.append(missing_reason) + + required_tag_gaps = [key for key in ("artist", "title") if not planned_tags.get(key)] + if required_tag_gaps and not suppress_review_flags: + group_review.append(f"Missing critical tags: {', '.join(sorted(required_tag_gaps))}.") + + group_confidence = "High (identical fingerprint cluster)" + if missing_fingerprints and not suppress_review_flags: + group_confidence = "Low (missing fingerprints in cluster)" + group_review.append("One or more tracks missing fingerprints; requires review.") + near_distances = [d for d in distance_samples if d > exact_duplicate_threshold] + if near_distances and not suppress_review_flags: + group_confidence = f"Medium (near-duplicate fingerprint distance up to {max_distance:.3f})" group_review.append( - f"Fingerprint distances exceed near-duplicate threshold ({near_duplicate_threshold:.3f}); grouping may be unsafe." + f"Audio match requires review (max distance {max_distance:.3f}; near-duplicate threshold {near_duplicate_threshold:.3f})." ) + if max_distance > near_duplicate_threshold and not suppress_review_flags: + group_confidence = f"Low (max fingerprint distance {max_distance:.3f})" + if not missing_fingerprints: + group_review.append( + f"Fingerprint distances exceed near-duplicate threshold ({near_duplicate_threshold:.3f}); grouping may be unsafe." + ) - dispositions = {loser: "quarantine" for loser in losers} - playlist_map = {loser: winner.path for loser in losers} - - group_id = _stable_group_id([t.path for t in group_tracks]) - if ambiguous_art: - group_review.append("Artwork selection requires review.") - if not losers: - group_review.append("No losers to consolidate.") - if any(_has_review_keyword(t) for t in group_tracks): - group_review.append("Contains remix/sped-up variant indicators; review recommended.") - placeholders = _placeholder_present(planned_tags) or any( - _placeholder_present(t.current_tags) for t in group_tracks - ) - if placeholders: - group_review.append("Placeholder metadata detected; requires review.") - plan_placeholders = True - if winner.metadata_error: - group_review.append(f"Metadata read issue for winner: {winner.metadata_error}") - if any(t.artwork_error for t in group_tracks): - group_review.append("Artwork extraction failed for at least one track.") - - playlist_impact = PlaylistImpact(playlists=len(losers), entries=len(losers)) - track_states = {t.path: dict(t.library_state) for t in group_tracks} - - artwork_candidates: List[ArtworkCandidate] = [] - for t in group_tracks: - artwork_candidates.extend(t.artwork) - - current_tags = {t.path: _merge_tags(_blank_tags(), t.current_tags) for t in group_tracks} - - plans.append( - GroupPlan( - group_id=group_id, - winner_path=winner.path, - losers=losers, - planned_winner_tags=planned_tags, - winner_current_tags=_merge_tags(_blank_tags(), winner.current_tags), - current_tags=current_tags, - metadata_changes=meta_changes, - winner_quality=winner_quality, - artwork=artwork_actions, - artwork_candidates=artwork_candidates, - chosen_artwork_source=chosen_artwork_source, - artwork_status=artwork_status, - loser_disposition=dispositions, - playlist_rewrites=playlist_map, - playlist_impact=playlist_impact, - review_flags=group_review, - context_summary={ - "album": sorted([p for p in group_paths if group_contexts.get(p) == "album"]), - "single": sorted([p for p in group_paths if group_contexts.get(p) == "single"]), - "unknown": sorted([p for p in group_paths if group_contexts.get(p) == "unknown"]), - }, - context_evidence=group_context_evidence, - tag_source=tag_source, - placeholders_present=placeholders, - tag_source_reason=tag_source_reason, - tag_source_evidence=tag_source_evidence, - track_quality=track_quality, - group_confidence=group_confidence, - group_match_type="Exact" if max_distance <= exact_duplicate_threshold else "Near-duplicate", - grouping_metadata_key=cluster.metadata_key, - grouping_thresholds={ - "exact": cluster.exact_threshold, - "near": cluster.near_threshold, - }, - grouping_decisions=group_decisions, - artwork_evidence=artwork_evidence, - fingerprint_distances=pair_distances, - library_state=track_states, + dispositions = {loser: "quarantine" for loser in losers} + playlist_map = {loser: winner.path for loser in losers} + + group_id = _stable_group_id([t.path for t in group_tracks]) + if ambiguous_art and not suppress_review_flags: + group_review.append("Artwork selection requires review.") + if not losers and not suppress_review_flags: + group_review.append("No losers to consolidate.") + if any(_has_review_keyword(t) for t in group_tracks) and not suppress_review_flags: + group_review.append("Contains remix/sped-up variant indicators; review recommended.") + placeholders = _placeholder_present(planned_tags) or any( + _placeholder_present(t.current_tags) for t in group_tracks + ) + if placeholders and not suppress_review_flags: + group_review.append("Placeholder metadata detected; requires review.") + plan_placeholders = True + if winner.metadata_error and not suppress_review_flags: + group_review.append(f"Metadata read issue for winner: {winner.metadata_error}") + if any(t.artwork_error for t in group_tracks) and not suppress_review_flags: + group_review.append("Artwork extraction failed for at least one track.") + + if suppress_review_flags: + group_review = [] + placeholders = False + + playlist_impact = PlaylistImpact(playlists=len(losers), entries=len(losers)) + track_states = {t.path: dict(t.library_state) for t in group_tracks} + + artwork_candidates: List[ArtworkCandidate] = [] + for t in group_tracks: + artwork_candidates.extend(t.artwork) + + current_tags = {t.path: _merge_tags(_blank_tags(), t.current_tags) for t in group_tracks} + + plans.append( + GroupPlan( + group_id=group_id, + winner_path=winner.path, + losers=losers, + planned_winner_tags=planned_tags, + winner_current_tags=_merge_tags(_blank_tags(), winner.current_tags), + current_tags=current_tags, + metadata_changes=meta_changes, + winner_quality=winner_quality, + artwork=artwork_actions, + artwork_candidates=artwork_candidates, + chosen_artwork_source=chosen_artwork_source, + artwork_status=artwork_status, + loser_disposition=dispositions, + playlist_rewrites=playlist_map, + playlist_impact=playlist_impact, + review_flags=group_review, + context_summary={ + "album": sorted([p for p in group_paths if group_contexts.get(p) == "album"]), + "single": sorted([p for p in group_paths if group_contexts.get(p) == "single"]), + "unknown": sorted([p for p in group_paths if group_contexts.get(p) == "unknown"]), + }, + context_evidence=group_context_evidence, + tag_source=tag_source, + placeholders_present=placeholders, + tag_source_reason=tag_source_reason, + tag_source_evidence=tag_source_evidence, + track_quality=track_quality, + group_confidence=group_confidence, + group_match_type="Exact" if max_distance <= exact_duplicate_threshold else "Near-duplicate", + grouping_metadata_key=cluster.metadata_key, + grouping_thresholds={ + "exact": cluster.exact_threshold, + "near": cluster.near_threshold, + }, + grouping_decisions=group_decisions, + artwork_evidence=artwork_evidence, + fingerprint_distances=pair_distances, + library_state=track_states, + ) ) - ) if plan_placeholders and "Placeholder metadata detected; review required." not in review_flags: review_flags.append("Placeholder metadata detected; review required.") From cd69d9efc556486a7b963f060b4dfc01a65c16f7 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Thu, 1 Jan 2026 10:50:55 -0500 Subject: [PATCH 214/606] Handle missing artwork in consolidation gate --- duplicate_consolidation.py | 39 ++++++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index 19e5c23..f5b6e1c 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -353,7 +353,7 @@ def _artwork_tracks_similar( left_hash = hashes.get(left.path) right_hash = hashes.get(right.path) if left_hash is None or right_hash is None: - return True + return False return _hamming_distance(left_hash, right_hash) <= ARTWORK_SIMILARITY_THRESHOLD @@ -1535,6 +1535,8 @@ def build_consolidation_plan( plan_placeholders = False for cluster in sorted(clusters, key=lambda c: _stable_group_id([t.path for t in c.tracks])): cluster_tracks = cluster.tracks + cluster_has_missing_artwork = any(not t.artwork for t in cluster_tracks) + cluster_has_unreadable_artwork = any(t.artwork_error for t in cluster_tracks) artwork_groups, artwork_hashes = _split_by_artwork_similarity(cluster_tracks) artwork_split = len(artwork_groups) > 1 for group_tracks in sorted(artwork_groups, key=lambda g: _stable_group_id([t.path for t in g])): @@ -1643,9 +1645,19 @@ def build_consolidation_plan( ) if artwork_split: if len(group_tracks) == 1: - artwork_evidence.append( - "Artwork differs from other audio duplicates; preserving this track as an intentional variant." - ) + group_track = group_tracks[0] + if not group_track.artwork or group_track.artwork_error: + artwork_evidence.append( + "Missing or unreadable artwork treated as non-matching; preserving this track as an intentional variant." + ) + elif cluster_has_missing_artwork or cluster_has_unreadable_artwork: + artwork_evidence.append( + "Another duplicate lacks readable artwork; missing artwork treated as non-matching and this track is preserved." + ) + else: + artwork_evidence.append( + "Artwork differs from other audio duplicates; preserving this track as an intentional variant." + ) else: artwork_evidence.append( "Artwork similarity gate grouped only tracks with matching covers for consolidation." @@ -1960,6 +1972,13 @@ def _planned_actions(group: GroupPlan) -> List[Dict[str, object]]: ) return actions + def _missing_artwork_gate(group: GroupPlan) -> bool: + for note in group.artwork_evidence: + lowered = note.lower() + if "non-matching" in lowered and "artwork" in lowered and "missing" in lowered: + return True + return False + def _action_badges(group: GroupPlan, actions: List[Dict[str, object]]) -> List[str]: badges = [] meta_count = sum(1 for act in actions if act["step"] == "metadata") @@ -2569,10 +2588,18 @@ def _image_mime(payload: bytes) -> str: "" ) else: + no_action_note = "No planned operations for this group." + no_action_status = "review" + if _missing_artwork_gate(group): + no_action_note = ( + "No planned operations; missing or unreadable artwork was treated as non-matching." + ) + no_action_status = "ready" + status_class = _status_badge_class(no_action_status) html_lines.append( "none—" - "review" - "No planned operations for this group." + f"{esc(no_action_status)}" + f"{esc(no_action_note)}" ) html_lines.append("") html_lines.append("") From 454cb8b14ce93d32c3213e667385aed74d4db5c0 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Thu, 1 Jan 2026 11:28:32 -0500 Subject: [PATCH 215/606] Improve fingerprint tuning for mixed codecs --- audio_norm.py | 61 ++++++--- config.py | 31 ++++- duplicate_consolidation.py | 65 ++++++++- fingerprint_generator.py | 110 +++++++++++++-- library_sync.py | 126 +++++++++++++++--- .../indexer_engine/config.py | 31 ++++- .../indexer_engine/fingerprint_generator.py | 110 +++++++++++++-- .../indexer_engine/music_indexer_api.py | 7 +- .../indexer_engine/near_duplicate_detector.py | 57 +++++++- library_sync_review.py | 11 +- main_gui.py | 40 +++++- music_indexer_api.py | 7 +- near_duplicate_detector.py | 57 +++++++- simple_duplicate_finder.py | 19 ++- tests/test_fingerprint_generator.py | 2 +- tests/test_library_sync.py | 8 +- tests/test_library_sync_integration.py | 2 +- 17 files changed, 648 insertions(+), 96 deletions(-) diff --git a/audio_norm.py b/audio_norm.py index e61cbc7..d9e6444 100644 --- a/audio_norm.py +++ b/audio_norm.py @@ -16,6 +16,9 @@ ) +SILENCE_THRESH = -50 # dBFS used for silence detection + + class _FallbackSegment: def __init__( self, @@ -125,21 +128,25 @@ def _safe_from_file(source, *args, **kwargs): AudioSegment.from_file = staticmethod(_safe_from_file) # type: ignore[assignment] -def _with_thresh(func, *args, **kwargs): +def _with_thresh(func, *args, silence_threshold_db: float = SILENCE_THRESH, **kwargs): """Call ``func`` with the correct silence threshold argument.""" try: - return func(*args, silence_threshold=SILENCE_THRESH, **kwargs) + return func(*args, silence_threshold=silence_threshold_db, **kwargs) except TypeError: - return func(*args, silence_thresh=SILENCE_THRESH, **kwargs) - - -SILENCE_THRESH = -50 # dBFS used for silence detection + return func(*args, silence_thresh=silence_threshold_db, **kwargs) def normalize_for_fp( path: str, fingerprint_offset_ms: int = 0, fingerprint_duration_ms: int = 120_000, + *, + trim_silence: bool = True, + silence_threshold_db: float = SILENCE_THRESH, + silence_min_len_ms: int = 50, + trim_padding_ms: int = 100, + trim_lead_max_ms: int = 500, + trim_trail_max_ms: int = 500, allow_mismatched_edits: bool = True, log_callback: Callable[[str], None] | None = None, ) -> io.BytesIO: @@ -149,22 +156,32 @@ def normalize_for_fp( audio = _FallbackSegment.from_file(path) audio = audio.set_frame_rate(44100).set_channels(2) - lead = _with_thresh(silence.detect_leading_silence, audio) - trim_lead = 0 - if lead > 100: - trim_lead = min(lead - 100, 500) - trail = 0 - end_sil = _with_thresh(silence.detect_silence, audio, min_silence_len=50) - if end_sil: - last_start, last_end = end_sil[-1] - if last_end >= len(audio): - trail = len(audio) - last_start - trim_trail = 0 - if trail > 100: - trim_trail = min(trail - 100, 500) - - if trim_lead or trim_trail: - audio = audio[trim_lead: len(audio) - trim_trail] + if trim_silence: + lead = _with_thresh( + silence.detect_leading_silence, + audio, + silence_threshold_db=silence_threshold_db, + ) + trim_lead = 0 + if lead > trim_padding_ms: + trim_lead = min(lead - trim_padding_ms, trim_lead_max_ms) + trail = 0 + end_sil = _with_thresh( + silence.detect_silence, + audio, + min_silence_len=silence_min_len_ms, + silence_threshold_db=silence_threshold_db, + ) + if end_sil: + last_start, last_end = end_sil[-1] + if last_end >= len(audio): + trail = len(audio) - last_start + trim_trail = 0 + if trail > trim_padding_ms: + trim_trail = min(trail - trim_padding_ms, trim_trail_max_ms) + + if trim_lead or trim_trail: + audio = audio[trim_lead: len(audio) - trim_trail] trimmed_len = len(audio) if abs(trimmed_len - fingerprint_duration_ms) > 0: diff --git a/config.py b/config.py index ac9abad..736e50b 100644 --- a/config.py +++ b/config.py @@ -14,6 +14,8 @@ # Threshold for considering two tracks as near-duplicates when syncing NEAR_DUPLICATE_THRESHOLD = 0.1 +EXACT_DUPLICATE_THRESHOLD = 0.02 +MIXED_CODEC_THRESHOLD_BOOST = 0.03 # File format quality priority used during Library Sync FORMAT_PRIORITY = {".flac": 3, ".wav": 2, ".mp3": 1} @@ -23,8 +25,9 @@ DEFAULT_FP_THRESHOLDS = { "default": 0.3, ".flac": 0.3, - ".mp3": 0.3, - ".aac": 0.3, + ".mp3": 0.35, + ".m4a": 0.35, + ".aac": 0.35, } # Default threshold for simple duplicate detection @@ -35,6 +38,12 @@ # Default settings for fingerprint normalization FP_OFFSET_MS = 0 FP_DURATION_MS = 120_000 +FP_TRIM_SILENCE = True +FP_SILENCE_THRESHOLD_DB = -50.0 +FP_SILENCE_MIN_LEN_MS = 500 +FP_TRIM_LEAD_MAX_MS = 1000 +FP_TRIM_TRAIL_MAX_MS = 1000 +FP_TRIM_PADDING_MS = 100 ALLOW_MISMATCHED_EDITS = True @@ -56,23 +65,41 @@ def load_config(): cfg["format_fp_thresholds"] = DEFAULT_FP_THRESHOLDS.copy() cfg.setdefault("duplicate_threshold", DEFAULT_DUP_THRESHOLD) cfg.setdefault("duplicate_prefix_len", DEFAULT_DUP_PREFIX_LEN) + cfg.setdefault("trim_silence", FP_TRIM_SILENCE) + cfg.setdefault("near_duplicate_threshold", NEAR_DUPLICATE_THRESHOLD) + cfg.setdefault("exact_duplicate_threshold", EXACT_DUPLICATE_THRESHOLD) cfg.setdefault("fingerprint_offset_ms", FP_OFFSET_MS) cfg.setdefault("fingerprint_duration_ms", FP_DURATION_MS) + cfg.setdefault("fingerprint_silence_threshold_db", FP_SILENCE_THRESHOLD_DB) + cfg.setdefault("fingerprint_silence_min_len_ms", FP_SILENCE_MIN_LEN_MS) + cfg.setdefault("fingerprint_trim_lead_max_ms", FP_TRIM_LEAD_MAX_MS) + cfg.setdefault("fingerprint_trim_trail_max_ms", FP_TRIM_TRAIL_MAX_MS) + cfg.setdefault("fingerprint_trim_padding_ms", FP_TRIM_PADDING_MS) cfg.setdefault("allow_mismatched_edits", ALLOW_MISMATCHED_EDITS) + cfg.setdefault("mixed_codec_threshold_boost", MIXED_CODEC_THRESHOLD_BOOST) cfg.setdefault("library_root", "") cfg.setdefault("use_library_sync_review", False) cfg.setdefault("library_sync_review", {}) return cfg except Exception: return { + "trim_silence": FP_TRIM_SILENCE, + "near_duplicate_threshold": NEAR_DUPLICATE_THRESHOLD, + "exact_duplicate_threshold": EXACT_DUPLICATE_THRESHOLD, "fingerprint_offset_ms": FP_OFFSET_MS, "fingerprint_duration_ms": FP_DURATION_MS, + "fingerprint_silence_threshold_db": FP_SILENCE_THRESHOLD_DB, + "fingerprint_silence_min_len_ms": FP_SILENCE_MIN_LEN_MS, + "fingerprint_trim_lead_max_ms": FP_TRIM_LEAD_MAX_MS, + "fingerprint_trim_trail_max_ms": FP_TRIM_TRAIL_MAX_MS, + "fingerprint_trim_padding_ms": FP_TRIM_PADDING_MS, "allow_mismatched_edits": ALLOW_MISMATCHED_EDITS, "library_root": "", "duplicate_threshold": DEFAULT_DUP_THRESHOLD, "duplicate_prefix_len": DEFAULT_DUP_PREFIX_LEN, "use_library_sync_review": False, "library_sync_review": {}, + "mixed_codec_threshold_boost": MIXED_CODEC_THRESHOLD_BOOST, } diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index f5b6e1c..962b7a4 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -845,6 +845,8 @@ class ConsolidationPlan: generated_at: datetime.datetime = field(default_factory=lambda: datetime.datetime.now(datetime.timezone.utc)) placeholders_present: bool = False source_snapshot: Dict[str, Dict[str, object]] = field(default_factory=dict) + fingerprint_settings: Dict[str, object] = field(default_factory=dict) + threshold_settings: Dict[str, float] = field(default_factory=dict) plan_signature: str | None = None def to_dict(self) -> Dict[str, object]: @@ -854,6 +856,8 @@ def to_dict(self) -> Dict[str, object]: "generated_at": self.generated_at.isoformat(), "placeholders_present": self.placeholders_present, "source_snapshot": {k: dict(v) for k, v in self.source_snapshot.items()}, + "fingerprint_settings": dict(self.fingerprint_settings), + "threshold_settings": dict(self.threshold_settings), "plan_signature": self.plan_signature, } @@ -888,6 +892,8 @@ def refresh_plan_signature(self) -> str: "generated_at": self.generated_at.isoformat(), "groups": [g.to_dict() for g in self.groups], "snapshot": {k: dict(v) for k, v in self.source_snapshot.items()}, + "fingerprint_settings": dict(self.fingerprint_settings), + "threshold_settings": dict(self.threshold_settings), }, sort_keys=True, ) @@ -1053,6 +1059,8 @@ def consolidation_plan_from_dict(raw: Mapping[str, object]) -> ConsolidationPlan generated_at = _expect_datetime(raw.get("generated_at"), "generated_at") placeholders_present = _expect_bool(raw.get("placeholders_present"), "placeholders_present") source_snapshot = _expect_mapping(raw.get("source_snapshot"), "source_snapshot") + fingerprint_settings = raw.get("fingerprint_settings") if isinstance(raw, Mapping) else None + threshold_settings = raw.get("threshold_settings") if isinstance(raw, Mapping) else None plan_signature = raw.get("plan_signature") if plan_signature is not None and not isinstance(plan_signature, str): raise ValueError("plan_signature must be a string or null.") @@ -1064,6 +1072,8 @@ def consolidation_plan_from_dict(raw: Mapping[str, object]) -> ConsolidationPlan generated_at=generated_at, placeholders_present=placeholders_present, source_snapshot=source_snapshot, + fingerprint_settings=dict(fingerprint_settings or {}), + threshold_settings={str(k): float(v) for k, v in dict(threshold_settings or {}).items()}, plan_signature=plan_signature, ) @@ -1370,6 +1380,7 @@ def _cluster_duplicates( *, exact_duplicate_threshold: float, near_duplicate_threshold: float, + mixed_codec_threshold_boost: float, cancel_event: threading.Event, max_comparisons: int, timeout_sec: float, @@ -1439,7 +1450,10 @@ def _cluster_duplicates( comparisons += 1 other = path_to_track[other_path] dist = fingerprint_distance(track.fingerprint, other.fingerprint) - if dist > near_duplicate_threshold: + pair_threshold = near_duplicate_threshold + if track.is_lossless != other.is_lossless: + pair_threshold += mixed_codec_threshold_boost + if dist > pair_threshold: continue compatible = True max_candidate_distance = dist @@ -1448,7 +1462,10 @@ def _cluster_duplicates( continue cmp_dist = fingerprint_distance(member.fingerprint, other.fingerprint) max_candidate_distance = max(max_candidate_distance, cmp_dist) - if cmp_dist > near_duplicate_threshold: + member_threshold = near_duplicate_threshold + if member.is_lossless != other.is_lossless: + member_threshold += mixed_codec_threshold_boost + if cmp_dist > member_threshold: compatible = False break if compatible: @@ -1467,7 +1484,7 @@ def _cluster_duplicates( shared_coarse_keys=shared_keys, distance_to_anchor=dist, max_group_distance=max_candidate_distance, - threshold=near_duplicate_threshold, + threshold=pair_threshold, match_type=match_type, coarse_gate=coarse_gate, ) @@ -1496,11 +1513,14 @@ def build_consolidation_plan( *, exact_duplicate_threshold: float = EXACT_DUPLICATE_THRESHOLD, near_duplicate_threshold: float = NEAR_DUPLICATE_THRESHOLD, + mixed_codec_threshold_boost: float = 0.0, max_candidates: int = DEFAULT_MAX_CANDIDATES, max_comparisons: int = DEFAULT_MAX_COMPARISONS, timeout_sec: float = DEFAULT_TIMEOUT_SEC, cancel_event: Optional[threading.Event] = None, progress_callback: Callable[[int, int, str], None] | None = None, + fingerprint_settings: Mapping[str, object] | None = None, + threshold_settings: Mapping[str, float] | None = None, ) -> ConsolidationPlan: """Generate a deterministic consolidation plan without modifying files.""" @@ -1523,6 +1543,7 @@ def build_consolidation_plan( normalized, exact_duplicate_threshold=exact_duplicate_threshold, near_duplicate_threshold=near_duplicate_threshold, + mixed_codec_threshold_boost=mixed_codec_threshold_boost, cancel_event=cancel_event, max_comparisons=max_comparisons, timeout_sec=timeout_sec, @@ -1856,6 +1877,7 @@ def build_consolidation_plan( grouping_thresholds={ "exact": cluster.exact_threshold, "near": cluster.near_threshold, + "mixed_codec_boost": mixed_codec_threshold_boost, }, grouping_decisions=group_decisions, artwork_evidence=artwork_evidence, @@ -1867,7 +1889,13 @@ def build_consolidation_plan( if plan_placeholders and "Placeholder metadata detected; review required." not in review_flags: review_flags.append("Placeholder metadata detected; review required.") - return ConsolidationPlan(groups=plans, review_flags=review_flags, placeholders_present=plan_placeholders) + return ConsolidationPlan( + groups=plans, + review_flags=review_flags, + placeholders_present=plan_placeholders, + fingerprint_settings=dict(fingerprint_settings or {}), + threshold_settings=dict(threshold_settings or {}), + ) def export_consolidation_preview(plan: ConsolidationPlan, output_json_path: str) -> str: @@ -2059,6 +2087,17 @@ def _image_mime(payload: bytes) -> str: joined_flags = "; ".join(plan.review_flags) global_notes = f"{global_notes} Review flags: {joined_flags}" + settings_entries: List[tuple[str, object]] = [] + if plan.threshold_settings: + settings_entries.extend(sorted(plan.threshold_settings.items(), key=lambda item: str(item[0]))) + if plan.fingerprint_settings: + settings_entries.extend(sorted(plan.fingerprint_settings.items(), key=lambda item: str(item[0]))) + + def _format_setting(value: object) -> str: + if isinstance(value, float): + return f"{value:.3f}" + return str(value) + html_lines = [ "", "", @@ -2424,6 +2463,24 @@ def _image_mime(payload: bytes) -> str: f"
      {esc(global_notes)}
      ", "
      ", "
      ", + ] + + if settings_entries: + html_lines.extend( + [ + "
      ", + "
      ", + "
      Fingerprint settings
      ", + "
      Thresholds + normalization inputs used for this plan.
      ", + ] + ) + for key, value in settings_entries: + html_lines.append(f"
      {esc(str(key))}
      ") + html_lines.append(f"
      {esc(_format_setting(value))}
      ") + html_lines.extend(["
      ", "
      "]) + + html_lines.extend( + [ "", "
      ", "
      ", diff --git a/fingerprint_generator.py b/fingerprint_generator.py index fb5d5ba..06e4387 100644 --- a/fingerprint_generator.py +++ b/fingerprint_generator.py @@ -9,24 +9,30 @@ SILENCE_THRESH = -50 # dBFS used for silence detection -def _with_thresh(func, *args, **kwargs): +def _with_thresh(func, *args, silence_threshold_db: float = SILENCE_THRESH, **kwargs): try: - return func(*args, silence_threshold=SILENCE_THRESH, **kwargs) + return func(*args, silence_threshold=silence_threshold_db, **kwargs) except TypeError: - return func(*args, silence_thresh=SILENCE_THRESH, **kwargs) + return func(*args, silence_thresh=silence_threshold_db, **kwargs) from concurrent.futures import ProcessPoolExecutor import acoustid from utils.path_helpers import ensure_long_path +import audio_norm SUPPORTED_EXTS = {".flac", ".m4a", ".aac", ".mp3", ".wav", ".ogg"} -def _trim_silence(path: str) -> str: +def _trim_silence(path: str, *, silence_threshold_db: float, min_silence_len_ms: int) -> str: """Return path to temporary file with leading/trailing silence removed.""" try: ext = os.path.splitext(path)[1].lower().lstrip(".") or "wav" audio = AudioSegment.from_file(path) - segs = _with_thresh(silence.detect_nonsilent, audio, min_silence_len=500) + segs = _with_thresh( + silence.detect_nonsilent, + audio, + min_silence_len=min_silence_len_ms, + silence_threshold_db=silence_threshold_db, + ) if not segs: return path start = segs[0][0] @@ -40,17 +46,48 @@ def _trim_silence(path: str) -> str: return path -def compute_fingerprint_for_file(args: Tuple[str, str, bool]) -> Tuple[str, int | None, str | None, str | None]: +def compute_fingerprint_for_file( + args: Tuple[str, str, dict], +) -> Tuple[str, int | None, str | None, str | None]: """Compute fingerprint for a single file. Returns (path, duration, fp, error).""" - path, _db_path, trim = args + path, _db_path, settings = args path = ensure_long_path(path) tmp = None try: - target = path - if trim: - tmp = _trim_silence(path) - target = tmp - duration, fp_hash = acoustid.fingerprint_file(target) + trim = bool(settings.get("trim_silence", False)) + silence_threshold_db = float(settings.get("silence_threshold_db", SILENCE_THRESH)) + silence_min_len_ms = int(settings.get("silence_min_len_ms", 500)) + fingerprint_offset_ms = int(settings.get("fingerprint_offset_ms", 0)) + fingerprint_duration_ms = int(settings.get("fingerprint_duration_ms", 0)) + allow_mismatched_edits = bool(settings.get("allow_mismatched_edits", True)) + trim_lead_max_ms = int(settings.get("trim_lead_max_ms", 500)) + trim_trail_max_ms = int(settings.get("trim_trail_max_ms", 500)) + trim_padding_ms = int(settings.get("trim_padding_ms", 100)) + + if fingerprint_duration_ms > 0: + buf = audio_norm.normalize_for_fp( + path, + fingerprint_offset_ms=fingerprint_offset_ms, + fingerprint_duration_ms=fingerprint_duration_ms, + trim_silence=trim, + silence_threshold_db=silence_threshold_db, + silence_min_len_ms=silence_min_len_ms, + trim_padding_ms=trim_padding_ms, + trim_lead_max_ms=trim_lead_max_ms, + trim_trail_max_ms=trim_trail_max_ms, + allow_mismatched_edits=allow_mismatched_edits, + ) + duration, fp_hash = acoustid.fingerprint_file(fileobj=buf) + else: + target = path + if trim: + tmp = _trim_silence( + path, + silence_threshold_db=silence_threshold_db, + min_silence_len_ms=silence_min_len_ms, + ) + target = tmp + duration, fp_hash = acoustid.fingerprint_file(target) if tmp and tmp != path: os.remove(tmp) return path, duration, fp_hash, None @@ -99,6 +136,14 @@ def compute_fingerprints_parallel( trim_silence: bool = False, phase: str = "A", cancel_event: threading.Event | None = None, + fingerprint_offset_ms: int = 0, + fingerprint_duration_ms: int = 0, + allow_mismatched_edits: bool = True, + silence_threshold_db: float = SILENCE_THRESH, + silence_min_len_ms: int = 500, + trim_lead_max_ms: int = 500, + trim_trail_max_ms: int = 500, + trim_padding_ms: int = 100, ) -> list[tuple[str, int | None, str | None]]: """Compute fingerprints and return ``(path, duration, fingerprint)`` results. @@ -139,6 +184,29 @@ def progress_callback(current: int, total: int, msg: str, _phase: str) -> None: total = len(audio_files) progress_callback(0, total, "Fingerprinting", phase) + fp_settings = { + "trim_silence": bool(trim_silence), + "silence_threshold_db": float(silence_threshold_db), + "silence_min_len_ms": int(silence_min_len_ms), + "fingerprint_offset_ms": int(fingerprint_offset_ms), + "fingerprint_duration_ms": int(fingerprint_duration_ms), + "allow_mismatched_edits": bool(allow_mismatched_edits), + "trim_lead_max_ms": int(trim_lead_max_ms), + "trim_trail_max_ms": int(trim_trail_max_ms), + "trim_padding_ms": int(trim_padding_ms), + } + if log_callback: + log_callback( + f"Fingerprint settings: trim_silence={fp_settings['trim_silence']} " + f"offset_ms={fp_settings['fingerprint_offset_ms']} duration_ms={fp_settings['fingerprint_duration_ms']} " + f"silence_threshold_db={fp_settings['silence_threshold_db']} " + f"silence_min_len_ms={fp_settings['silence_min_len_ms']} " + f"trim_lead_max_ms={fp_settings['trim_lead_max_ms']} " + f"trim_trail_max_ms={fp_settings['trim_trail_max_ms']} " + f"trim_padding_ms={fp_settings['trim_padding_ms']} " + f"allow_mismatched_edits={fp_settings['allow_mismatched_edits']}" + ) + cached = _load_cached_entries(conn) pending: list[str] = [] completed = 0 @@ -157,7 +225,7 @@ def progress_callback(current: int, total: int, msg: str, _phase: str) -> None: continue pending.append(path) - work_items = [(p, db_path, trim_silence) for p in pending] + work_items = [(p, db_path, fp_settings) for p in pending] if pending: with ProcessPoolExecutor(max_workers=max_workers) as exe: @@ -210,6 +278,14 @@ def compute_fingerprints( trim_silence: bool = False, phase: str = "A", cancel_event: threading.Event | None = None, + fingerprint_offset_ms: int = 0, + fingerprint_duration_ms: int = 0, + allow_mismatched_edits: bool = True, + silence_threshold_db: float = SILENCE_THRESH, + silence_min_len_ms: int = 500, + trim_lead_max_ms: int = 500, + trim_trail_max_ms: int = 500, + trim_padding_ms: int = 100, ) -> None: """Backward-compatible wrapper around :func:`compute_fingerprints_parallel`.""" return compute_fingerprints_parallel( @@ -221,4 +297,12 @@ def compute_fingerprints( trim_silence, phase, cancel_event, + fingerprint_offset_ms, + fingerprint_duration_ms, + allow_mismatched_edits, + silence_threshold_db, + silence_min_len_ms, + trim_lead_max_ms, + trim_trail_max_ms, + trim_padding_ms, ) diff --git a/library_sync.py b/library_sync.py index 4ce956f..754d5e3 100644 --- a/library_sync.py +++ b/library_sync.py @@ -28,6 +28,7 @@ _ensure_db = idx_fingerprint_cache._ensure_db FORMAT_PRIORITY = idx_config.FORMAT_PRIORITY DEFAULT_FP_THRESHOLDS = idx_config.DEFAULT_FP_THRESHOLDS +MIXED_CODEC_THRESHOLD_BOOST = idx_config.MIXED_CODEC_THRESHOLD_BOOST from crash_watcher import record_event from crash_logger import watcher @@ -43,6 +44,7 @@ NEAR_MISS_MARGIN_FLOOR = 0.02 SIGNATURE_TOKEN_COUNT = 8 SHORTLIST_FALLBACK = 50 +LOSSLESS_EXTS = {".flac", ".wav", ".alac", ".ape", ".aiff", ".aif"} def set_debug(enabled: bool, log_root: str | None = None) -> None: @@ -255,6 +257,37 @@ def _select_threshold(ext: str, thresholds: Dict[str, float]) -> float: return float(thresholds.get(ext, thresholds.get("default", 0.3))) +def _select_pair_threshold( + ext_a: str, + ext_b: str, + thresholds: Dict[str, float], + mixed_codec_boost: float, +) -> float: + base = max(_select_threshold(ext_a, thresholds), _select_threshold(ext_b, thresholds)) + if ext_a and ext_b and ext_a != ext_b: + lossless_a = ext_a in LOSSLESS_EXTS + lossless_b = ext_b in LOSSLESS_EXTS + if lossless_a != lossless_b: + base += mixed_codec_boost + return base + + +def _fingerprint_settings_from_config(cfg: Dict[str, object]) -> Dict[str, object]: + return { + "trim_silence": bool(cfg.get("trim_silence", idx_config.FP_TRIM_SILENCE)), + "fingerprint_offset_ms": int(cfg.get("fingerprint_offset_ms", idx_config.FP_OFFSET_MS)), + "fingerprint_duration_ms": int(cfg.get("fingerprint_duration_ms", idx_config.FP_DURATION_MS)), + "allow_mismatched_edits": bool(cfg.get("allow_mismatched_edits", idx_config.ALLOW_MISMATCHED_EDITS)), + "silence_threshold_db": float( + cfg.get("fingerprint_silence_threshold_db", idx_config.FP_SILENCE_THRESHOLD_DB) + ), + "silence_min_len_ms": int(cfg.get("fingerprint_silence_min_len_ms", idx_config.FP_SILENCE_MIN_LEN_MS)), + "trim_lead_max_ms": int(cfg.get("fingerprint_trim_lead_max_ms", idx_config.FP_TRIM_LEAD_MAX_MS)), + "trim_trail_max_ms": int(cfg.get("fingerprint_trim_trail_max_ms", idx_config.FP_TRIM_TRAIL_MAX_MS)), + "trim_padding_ms": int(cfg.get("fingerprint_trim_padding_ms", idx_config.FP_TRIM_PADDING_MS)), + } + + def compute_quality_score(info: Dict[str, object], fmt_priority: Dict[str, int]) -> int: pr = fmt_priority.get(info.get("ext"), 0) bitrate_val = info.get("bitrate") @@ -287,6 +320,7 @@ def _scan_folder( progress_callback: Callable[[int, int, str, str], None] | None = None, cancel_event: threading.Event | None = None, profiler: PerformanceProfile | None = None, + fingerprint_settings: Dict[str, object] | None = None, ) -> List[TrackRecord]: """Enumerate audio files, normalize paths, and attach cached fingerprints.""" cancel_event = cancel_event or threading.Event() @@ -391,10 +425,32 @@ def fp_progress(current: int, total: int, path: str, phase: str) -> None: ) if to_compute: + if fingerprint_settings is None: + cfg = idx_config.load_config() + fingerprint_settings = _fingerprint_settings_from_config(cfg) + if log_callback: + log_callback( + json.dumps({"event": "fingerprint_settings", "settings": dict(fingerprint_settings)}) + ) if log_callback: log_callback(json.dumps({"event": "fingerprint_start", "pending": len(to_compute), "folder": folder})) for result in fingerprint_generator.compute_fingerprints_parallel( - to_compute, db_path, log_callback, fp_progress, cancel_event=cancel_event + to_compute, + db_path, + log_callback, + fp_progress, + cancel_event=cancel_event, + trim_silence=bool(fingerprint_settings.get("trim_silence", False)), + fingerprint_offset_ms=int(fingerprint_settings.get("fingerprint_offset_ms", 0)), + fingerprint_duration_ms=int(fingerprint_settings.get("fingerprint_duration_ms", 0)), + allow_mismatched_edits=bool(fingerprint_settings.get("allow_mismatched_edits", True)), + silence_threshold_db=float( + fingerprint_settings.get("silence_threshold_db", idx_config.FP_SILENCE_THRESHOLD_DB) + ), + silence_min_len_ms=int(fingerprint_settings.get("silence_min_len_ms", 500)), + trim_lead_max_ms=int(fingerprint_settings.get("trim_lead_max_ms", 500)), + trim_trail_max_ms=int(fingerprint_settings.get("trim_trail_max_ms", 500)), + trim_padding_ms=int(fingerprint_settings.get("trim_padding_ms", 100)), ): path = result[0] duration = result[1] if len(result) > 2 else None @@ -483,6 +539,7 @@ def _match_tracks( library: List[TrackRecord], thresholds: Dict[str, float], fmt_priority: Dict[str, int], + mixed_codec_threshold_boost: float = MIXED_CODEC_THRESHOLD_BOOST, log_callback: Callable[[str], None] | None = None, profiler: PerformanceProfile | None = None, ) -> List[MatchResult]: @@ -491,30 +548,51 @@ def _match_tracks( results: List[MatchResult] = [] start_time = time.perf_counter() for inc in incoming: - threshold_used = _select_threshold(inc.ext, thresholds) - near_miss_margin = max(NEAR_MISS_MARGIN_FLOOR, threshold_used * NEAR_MISS_MARGIN_RATIO) candidates = _shortlist_candidates( inc, sig_index, ext_index, fallback_pool, log_callback=log_callback, profiler=profiler ) cand_dist: List[CandidateDistance] = [] + candidate_meta: List[Tuple[TrackRecord, float, float, float, MatchStatus]] = [] for cand in candidates: dist = fingerprint_distance(inc.fingerprint, cand.fingerprint) + threshold_for_pair = _select_pair_threshold( + inc.ext, cand.ext, thresholds, mixed_codec_threshold_boost + ) + near_miss_margin = max(NEAR_MISS_MARGIN_FLOOR, threshold_for_pair * NEAR_MISS_MARGIN_RATIO) + if dist == 0: + status = MatchStatus.EXACT_MATCH + elif dist <= threshold_for_pair: + status = MatchStatus.COLLISION + elif dist <= threshold_for_pair + near_miss_margin: + status = MatchStatus.LOW_CONFIDENCE + else: + status = MatchStatus.NEW + candidate_meta.append((cand, dist, threshold_for_pair, near_miss_margin, status)) cand_dist.append(CandidateDistance(distance=dist, track=cand)) cand_dist.sort() - best = cand_dist[0] if cand_dist else None - best_track = best.track if best else None - best_distance = best.distance if best else None + best_track = None + best_distance = None + threshold_used = _select_threshold(inc.ext, thresholds) + near_miss_margin = max(NEAR_MISS_MARGIN_FLOOR, threshold_used * NEAR_MISS_MARGIN_RATIO) + status = MatchStatus.NEW + best_rank = -1 + for cand, dist, thr, margin, cand_status in candidate_meta: + rank = { + MatchStatus.EXACT_MATCH: 3, + MatchStatus.COLLISION: 2, + MatchStatus.LOW_CONFIDENCE: 1, + MatchStatus.NEW: 0, + }[cand_status] + if best_track is None or rank > best_rank or (rank == best_rank and dist < (best_distance or 1)): + best_track = cand + best_distance = dist + threshold_used = thr + near_miss_margin = margin + status = cand_status + best_rank = rank if best_distance is None or best_track is None: status = MatchStatus.NEW - elif best_distance == 0: - status = MatchStatus.EXACT_MATCH - elif best_distance <= threshold_used: - status = MatchStatus.COLLISION - elif best_distance <= threshold_used + near_miss_margin: - status = MatchStatus.LOW_CONFIDENCE - else: - status = MatchStatus.NEW inc_score = compute_quality_score(inc.__dict__, fmt_priority) existing_score = compute_quality_score(best_track.__dict__, fmt_priority) if best_track else None @@ -528,7 +606,8 @@ def _match_tracks( confidence = _compute_confidence(best_distance, threshold_used, near_miss_margin) _dlog( f"DEBUG: match incoming={inc.path} best={best_track.path if best_track else 'none'} " - f"dist={best_distance} thr={threshold_used} margin={near_miss_margin} status={status} confidence={confidence}", + f"dist={best_distance} thr={threshold_used} margin={near_miss_margin} " + f"mixed_boost={mixed_codec_threshold_boost} status={status} confidence={confidence}", log_callback, ) results.append( @@ -559,6 +638,8 @@ def compare_libraries( threshold: float | None = None, fmt_priority: Dict[str, int] | None = None, thresholds: Dict[str, float] | None = None, + mixed_codec_threshold_boost: float | None = None, + fingerprint_settings: Dict[str, object] | None = None, log_callback: Callable[[str], None] | None = None, progress_callback: Callable[[int, int, str, str], None] | None = None, cancel_event: threading.Event | None = None, @@ -588,6 +669,11 @@ def compare_libraries( else: thresholds = DEFAULT_FP_THRESHOLDS fmt_priority = fmt_priority or FORMAT_PRIORITY + if mixed_codec_threshold_boost is None: + cfg = idx_config.load_config() + mixed_codec_threshold_boost = float( + cfg.get("mixed_codec_threshold_boost", MIXED_CODEC_THRESHOLD_BOOST) + ) start = time.perf_counter() lib_records = _scan_folder( @@ -597,6 +683,7 @@ def compare_libraries( progress_callback=progress_callback, cancel_event=cancel_event, profiler=profiler, + fingerprint_settings=fingerprint_settings, ) if profiler: profiler.library_scan_duration = time.perf_counter() - start @@ -608,12 +695,19 @@ def compare_libraries( progress_callback=progress_callback, cancel_event=cancel_event, profiler=profiler, + fingerprint_settings=fingerprint_settings, ) if profiler: profiler.incoming_scan_duration = time.perf_counter() - start match_results = _match_tracks( - inc_records, lib_records, thresholds, fmt_priority, log_callback=log_callback, profiler=profiler + inc_records, + lib_records, + thresholds, + fmt_priority, + mixed_codec_threshold_boost=mixed_codec_threshold_boost, + log_callback=log_callback, + profiler=profiler, ) new_paths: List[str] = [] diff --git a/library_sync_indexer_engine/indexer_engine/config.py b/library_sync_indexer_engine/indexer_engine/config.py index ac9abad..736e50b 100644 --- a/library_sync_indexer_engine/indexer_engine/config.py +++ b/library_sync_indexer_engine/indexer_engine/config.py @@ -14,6 +14,8 @@ # Threshold for considering two tracks as near-duplicates when syncing NEAR_DUPLICATE_THRESHOLD = 0.1 +EXACT_DUPLICATE_THRESHOLD = 0.02 +MIXED_CODEC_THRESHOLD_BOOST = 0.03 # File format quality priority used during Library Sync FORMAT_PRIORITY = {".flac": 3, ".wav": 2, ".mp3": 1} @@ -23,8 +25,9 @@ DEFAULT_FP_THRESHOLDS = { "default": 0.3, ".flac": 0.3, - ".mp3": 0.3, - ".aac": 0.3, + ".mp3": 0.35, + ".m4a": 0.35, + ".aac": 0.35, } # Default threshold for simple duplicate detection @@ -35,6 +38,12 @@ # Default settings for fingerprint normalization FP_OFFSET_MS = 0 FP_DURATION_MS = 120_000 +FP_TRIM_SILENCE = True +FP_SILENCE_THRESHOLD_DB = -50.0 +FP_SILENCE_MIN_LEN_MS = 500 +FP_TRIM_LEAD_MAX_MS = 1000 +FP_TRIM_TRAIL_MAX_MS = 1000 +FP_TRIM_PADDING_MS = 100 ALLOW_MISMATCHED_EDITS = True @@ -56,23 +65,41 @@ def load_config(): cfg["format_fp_thresholds"] = DEFAULT_FP_THRESHOLDS.copy() cfg.setdefault("duplicate_threshold", DEFAULT_DUP_THRESHOLD) cfg.setdefault("duplicate_prefix_len", DEFAULT_DUP_PREFIX_LEN) + cfg.setdefault("trim_silence", FP_TRIM_SILENCE) + cfg.setdefault("near_duplicate_threshold", NEAR_DUPLICATE_THRESHOLD) + cfg.setdefault("exact_duplicate_threshold", EXACT_DUPLICATE_THRESHOLD) cfg.setdefault("fingerprint_offset_ms", FP_OFFSET_MS) cfg.setdefault("fingerprint_duration_ms", FP_DURATION_MS) + cfg.setdefault("fingerprint_silence_threshold_db", FP_SILENCE_THRESHOLD_DB) + cfg.setdefault("fingerprint_silence_min_len_ms", FP_SILENCE_MIN_LEN_MS) + cfg.setdefault("fingerprint_trim_lead_max_ms", FP_TRIM_LEAD_MAX_MS) + cfg.setdefault("fingerprint_trim_trail_max_ms", FP_TRIM_TRAIL_MAX_MS) + cfg.setdefault("fingerprint_trim_padding_ms", FP_TRIM_PADDING_MS) cfg.setdefault("allow_mismatched_edits", ALLOW_MISMATCHED_EDITS) + cfg.setdefault("mixed_codec_threshold_boost", MIXED_CODEC_THRESHOLD_BOOST) cfg.setdefault("library_root", "") cfg.setdefault("use_library_sync_review", False) cfg.setdefault("library_sync_review", {}) return cfg except Exception: return { + "trim_silence": FP_TRIM_SILENCE, + "near_duplicate_threshold": NEAR_DUPLICATE_THRESHOLD, + "exact_duplicate_threshold": EXACT_DUPLICATE_THRESHOLD, "fingerprint_offset_ms": FP_OFFSET_MS, "fingerprint_duration_ms": FP_DURATION_MS, + "fingerprint_silence_threshold_db": FP_SILENCE_THRESHOLD_DB, + "fingerprint_silence_min_len_ms": FP_SILENCE_MIN_LEN_MS, + "fingerprint_trim_lead_max_ms": FP_TRIM_LEAD_MAX_MS, + "fingerprint_trim_trail_max_ms": FP_TRIM_TRAIL_MAX_MS, + "fingerprint_trim_padding_ms": FP_TRIM_PADDING_MS, "allow_mismatched_edits": ALLOW_MISMATCHED_EDITS, "library_root": "", "duplicate_threshold": DEFAULT_DUP_THRESHOLD, "duplicate_prefix_len": DEFAULT_DUP_PREFIX_LEN, "use_library_sync_review": False, "library_sync_review": {}, + "mixed_codec_threshold_boost": MIXED_CODEC_THRESHOLD_BOOST, } diff --git a/library_sync_indexer_engine/indexer_engine/fingerprint_generator.py b/library_sync_indexer_engine/indexer_engine/fingerprint_generator.py index fb5d5ba..06e4387 100644 --- a/library_sync_indexer_engine/indexer_engine/fingerprint_generator.py +++ b/library_sync_indexer_engine/indexer_engine/fingerprint_generator.py @@ -9,24 +9,30 @@ SILENCE_THRESH = -50 # dBFS used for silence detection -def _with_thresh(func, *args, **kwargs): +def _with_thresh(func, *args, silence_threshold_db: float = SILENCE_THRESH, **kwargs): try: - return func(*args, silence_threshold=SILENCE_THRESH, **kwargs) + return func(*args, silence_threshold=silence_threshold_db, **kwargs) except TypeError: - return func(*args, silence_thresh=SILENCE_THRESH, **kwargs) + return func(*args, silence_thresh=silence_threshold_db, **kwargs) from concurrent.futures import ProcessPoolExecutor import acoustid from utils.path_helpers import ensure_long_path +import audio_norm SUPPORTED_EXTS = {".flac", ".m4a", ".aac", ".mp3", ".wav", ".ogg"} -def _trim_silence(path: str) -> str: +def _trim_silence(path: str, *, silence_threshold_db: float, min_silence_len_ms: int) -> str: """Return path to temporary file with leading/trailing silence removed.""" try: ext = os.path.splitext(path)[1].lower().lstrip(".") or "wav" audio = AudioSegment.from_file(path) - segs = _with_thresh(silence.detect_nonsilent, audio, min_silence_len=500) + segs = _with_thresh( + silence.detect_nonsilent, + audio, + min_silence_len=min_silence_len_ms, + silence_threshold_db=silence_threshold_db, + ) if not segs: return path start = segs[0][0] @@ -40,17 +46,48 @@ def _trim_silence(path: str) -> str: return path -def compute_fingerprint_for_file(args: Tuple[str, str, bool]) -> Tuple[str, int | None, str | None, str | None]: +def compute_fingerprint_for_file( + args: Tuple[str, str, dict], +) -> Tuple[str, int | None, str | None, str | None]: """Compute fingerprint for a single file. Returns (path, duration, fp, error).""" - path, _db_path, trim = args + path, _db_path, settings = args path = ensure_long_path(path) tmp = None try: - target = path - if trim: - tmp = _trim_silence(path) - target = tmp - duration, fp_hash = acoustid.fingerprint_file(target) + trim = bool(settings.get("trim_silence", False)) + silence_threshold_db = float(settings.get("silence_threshold_db", SILENCE_THRESH)) + silence_min_len_ms = int(settings.get("silence_min_len_ms", 500)) + fingerprint_offset_ms = int(settings.get("fingerprint_offset_ms", 0)) + fingerprint_duration_ms = int(settings.get("fingerprint_duration_ms", 0)) + allow_mismatched_edits = bool(settings.get("allow_mismatched_edits", True)) + trim_lead_max_ms = int(settings.get("trim_lead_max_ms", 500)) + trim_trail_max_ms = int(settings.get("trim_trail_max_ms", 500)) + trim_padding_ms = int(settings.get("trim_padding_ms", 100)) + + if fingerprint_duration_ms > 0: + buf = audio_norm.normalize_for_fp( + path, + fingerprint_offset_ms=fingerprint_offset_ms, + fingerprint_duration_ms=fingerprint_duration_ms, + trim_silence=trim, + silence_threshold_db=silence_threshold_db, + silence_min_len_ms=silence_min_len_ms, + trim_padding_ms=trim_padding_ms, + trim_lead_max_ms=trim_lead_max_ms, + trim_trail_max_ms=trim_trail_max_ms, + allow_mismatched_edits=allow_mismatched_edits, + ) + duration, fp_hash = acoustid.fingerprint_file(fileobj=buf) + else: + target = path + if trim: + tmp = _trim_silence( + path, + silence_threshold_db=silence_threshold_db, + min_silence_len_ms=silence_min_len_ms, + ) + target = tmp + duration, fp_hash = acoustid.fingerprint_file(target) if tmp and tmp != path: os.remove(tmp) return path, duration, fp_hash, None @@ -99,6 +136,14 @@ def compute_fingerprints_parallel( trim_silence: bool = False, phase: str = "A", cancel_event: threading.Event | None = None, + fingerprint_offset_ms: int = 0, + fingerprint_duration_ms: int = 0, + allow_mismatched_edits: bool = True, + silence_threshold_db: float = SILENCE_THRESH, + silence_min_len_ms: int = 500, + trim_lead_max_ms: int = 500, + trim_trail_max_ms: int = 500, + trim_padding_ms: int = 100, ) -> list[tuple[str, int | None, str | None]]: """Compute fingerprints and return ``(path, duration, fingerprint)`` results. @@ -139,6 +184,29 @@ def progress_callback(current: int, total: int, msg: str, _phase: str) -> None: total = len(audio_files) progress_callback(0, total, "Fingerprinting", phase) + fp_settings = { + "trim_silence": bool(trim_silence), + "silence_threshold_db": float(silence_threshold_db), + "silence_min_len_ms": int(silence_min_len_ms), + "fingerprint_offset_ms": int(fingerprint_offset_ms), + "fingerprint_duration_ms": int(fingerprint_duration_ms), + "allow_mismatched_edits": bool(allow_mismatched_edits), + "trim_lead_max_ms": int(trim_lead_max_ms), + "trim_trail_max_ms": int(trim_trail_max_ms), + "trim_padding_ms": int(trim_padding_ms), + } + if log_callback: + log_callback( + f"Fingerprint settings: trim_silence={fp_settings['trim_silence']} " + f"offset_ms={fp_settings['fingerprint_offset_ms']} duration_ms={fp_settings['fingerprint_duration_ms']} " + f"silence_threshold_db={fp_settings['silence_threshold_db']} " + f"silence_min_len_ms={fp_settings['silence_min_len_ms']} " + f"trim_lead_max_ms={fp_settings['trim_lead_max_ms']} " + f"trim_trail_max_ms={fp_settings['trim_trail_max_ms']} " + f"trim_padding_ms={fp_settings['trim_padding_ms']} " + f"allow_mismatched_edits={fp_settings['allow_mismatched_edits']}" + ) + cached = _load_cached_entries(conn) pending: list[str] = [] completed = 0 @@ -157,7 +225,7 @@ def progress_callback(current: int, total: int, msg: str, _phase: str) -> None: continue pending.append(path) - work_items = [(p, db_path, trim_silence) for p in pending] + work_items = [(p, db_path, fp_settings) for p in pending] if pending: with ProcessPoolExecutor(max_workers=max_workers) as exe: @@ -210,6 +278,14 @@ def compute_fingerprints( trim_silence: bool = False, phase: str = "A", cancel_event: threading.Event | None = None, + fingerprint_offset_ms: int = 0, + fingerprint_duration_ms: int = 0, + allow_mismatched_edits: bool = True, + silence_threshold_db: float = SILENCE_THRESH, + silence_min_len_ms: int = 500, + trim_lead_max_ms: int = 500, + trim_trail_max_ms: int = 500, + trim_padding_ms: int = 100, ) -> None: """Backward-compatible wrapper around :func:`compute_fingerprints_parallel`.""" return compute_fingerprints_parallel( @@ -221,4 +297,12 @@ def compute_fingerprints( trim_silence, phase, cancel_event, + fingerprint_offset_ms, + fingerprint_duration_ms, + allow_mismatched_edits, + silence_threshold_db, + silence_min_len_ms, + trim_lead_max_ms, + trim_trail_max_ms, + trim_padding_ms, ) diff --git a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py index 449e611..0c9f645 100644 --- a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py +++ b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py @@ -8,7 +8,7 @@ from collections import defaultdict from typing import Dict, List from dry_run_coordinator import DryRunCoordinator -from config import load_config +from config import MIXED_CODEC_THRESHOLD_BOOST, load_config def _verify_dependencies() -> None: """Raise RuntimeError if the real mutagen library is missing.""" @@ -435,12 +435,14 @@ def _compute(path: str) -> tuple[int | None, str | None]: fp = fp.decode("utf-8") except Exception: fp = fp.decode("latin1", errors="ignore") + ext = os.path.splitext(fullpath)[1].lower() file_infos[fullpath] = { "primary": primary.lower(), "title": title.lower(), "album": album.lower(), "fp": fp, "meta_count": meta_count, + "ext": ext, } to_delete: Dict[str, str] = {} @@ -490,6 +492,8 @@ def _compute(path: str) -> tuple[int | None, str | None]: # --- Near-duplicate detection ------------------------------------------------- from near_duplicate_detector import find_near_duplicates log_callback(" • Detecting near-duplicates…") + cfg = load_config() + mixed_codec_boost = float(cfg.get("mixed_codec_threshold_boost", MIXED_CODEC_THRESHOLD_BOOST)) near_dupes = find_near_duplicates( {p: file_infos[p] for p in kept_files}, EXT_PRIORITY, @@ -498,6 +502,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: enable_phase_c, coord, max_workers, + mixed_codec_boost=mixed_codec_boost, ) if near_dupes.review_required: log_callback( diff --git a/library_sync_indexer_engine/indexer_engine/near_duplicate_detector.py b/library_sync_indexer_engine/indexer_engine/near_duplicate_detector.py index 22d4d4e..3901faf 100644 --- a/library_sync_indexer_engine/indexer_engine/near_duplicate_detector.py +++ b/library_sync_indexer_engine/indexer_engine/near_duplicate_detector.py @@ -21,6 +21,7 @@ COARSE_FP_BAND_SIZE = 8 COARSE_FP_BANDS = 6 COARSE_FP_QUANTIZATION = 4 +LOSSLESS_EXTS = {".flac", ".wav", ".alac", ".ape", ".aiff", ".aif"} @dataclass @@ -131,6 +132,34 @@ def _metadata_gate(info_a: Mapping[str, str | None], info_b: Mapping[str, str | return False +def _path_ext(path: str, info: Mapping[str, str | None]) -> str: + ext = info.get("ext") if isinstance(info, Mapping) else None + if ext: + return str(ext).lower() if str(ext).startswith(".") else f".{str(ext).lower()}" + return os.path.splitext(path)[1].lower() + + +def _is_lossless(ext: str) -> bool: + return ext in LOSSLESS_EXTS + + +def _pair_threshold( + path_a: str, + path_b: str, + info_a: Mapping[str, str | None], + info_b: Mapping[str, str | None], + base_threshold: float, + mixed_codec_boost: float, +) -> float: + ext_a = _path_ext(path_a, info_a) + ext_b = _path_ext(path_b, info_b) + if not ext_a or not ext_b: + return base_threshold + if _is_lossless(ext_a) != _is_lossless(ext_b): + return base_threshold + mixed_codec_boost + return base_threshold + + def _coarse_fingerprint_keys(fp: str | None) -> List[str]: parsed = _parse_fp(fp) if fp else None values: List[int] = [] @@ -177,6 +206,7 @@ def _scan_paths( paths: Iterable[str], file_infos: Dict[str, Dict[str, str | None]], threshold: float, + mixed_codec_boost: float, log_callback, ) -> Tuple[List[Tuple[Set[str], float]], int, int, int]: """Scan a list of paths for near duplicate clusters.""" @@ -196,12 +226,21 @@ def _scan_paths( coarse_blocked += 1 continue dist = fingerprint_distance(file_infos[p]["fp"], file_infos[q]["fp"]) - if dist <= threshold: + pair_threshold = _pair_threshold( + p, + q, + file_infos[p], + file_infos[q], + threshold, + mixed_codec_boost, + ) + if dist <= pair_threshold: adj[p].add(q) adj[q].add(p) pair_distances[frozenset({p, q})] = dist log_callback( - f" → near-dup {os.path.basename(p)} vs {os.path.basename(q)} dist={dist:.3f}" + f" → near-dup {os.path.basename(p)} vs {os.path.basename(q)} " + f"dist={dist:.3f} thr={pair_threshold:.3f}" ) comparisons += 1 @@ -233,10 +272,17 @@ def _scan_album( paths: Iterable[str], file_infos: Dict[str, Dict[str, str | None]], threshold: float, + mixed_codec_boost: float, log_callback, ) -> tuple[str, List[Tuple[Set[str], float]], int, int, int]: """Worker helper: scan a single album for near duplicates.""" - clusters, comparisons, meta_blocked, coarse_blocked = _scan_paths(paths, file_infos, threshold, log_callback) + clusters, comparisons, meta_blocked, coarse_blocked = _scan_paths( + paths, + file_infos, + threshold, + mixed_codec_boost, + log_callback, + ) return album, clusters, comparisons, meta_blocked, coarse_blocked @@ -248,6 +294,7 @@ def find_near_duplicates( enable_cross_album: bool = False, coord=None, max_workers: int | None = None, + mixed_codec_boost: float = 0.0, ) -> NearDuplicateResult: """Return reviewable near-duplicate groups (auto-merging is gated by review).""" if log_callback is None: @@ -278,7 +325,7 @@ def log_callback(msg: str) -> None: f"Phase B: [{idx}/{total_albums}] Scanning album '{album}' ({len(album_paths)} tracks)" ) html_lines.append(f"Album: {album or ''} ({len(album_paths)} tracks)") - futures[ex.submit(_scan_album, album, album_paths, file_infos, threshold, log_callback)] = album + futures[ex.submit(_scan_album, album, album_paths, file_infos, threshold, mixed_codec_boost, log_callback)] = album for fut in as_completed(futures): album = futures[fut] @@ -321,7 +368,7 @@ def log_callback(msg: str) -> None: f"Phase C: [{idx}/{total_songs}] Scanning '{title}' ({len(song_paths)} tracks)" ) cross_lines.append(f"{title or ''} ({len(song_paths)} tracks)") - futures[ex.submit(_scan_paths, song_paths, file_infos, threshold, log_callback)] = (prim, title) + futures[ex.submit(_scan_paths, song_paths, file_infos, threshold, mixed_codec_boost, log_callback)] = (prim, title) for fut in as_completed(futures): _key = futures[fut] diff --git a/library_sync_review.py b/library_sync_review.py index 25773fe..312c31f 100644 --- a/library_sync_review.py +++ b/library_sync_review.py @@ -1020,12 +1020,21 @@ def _maybe_match(self) -> None: overrides = self._parse_overrides() or {} global_thr = DEFAULT_FP_THRESHOLDS.get("default", 0.3) thresholds = {"default": global_thr, **overrides} - self._log_event("match_start", "Computing matches", {"thresholds": thresholds}) + cfg = load_config() + mixed_codec_boost = float( + cfg.get("mixed_codec_threshold_boost", library_sync.MIXED_CODEC_THRESHOLD_BOOST) + ) + self._log_event( + "match_start", + "Computing matches", + {"thresholds": thresholds, "mixed_codec_threshold_boost": mixed_codec_boost}, + ) results = library_sync._match_tracks( self.incoming_records, self.library_records, thresholds, FORMAT_PRIORITY, + mixed_codec_threshold_boost=mixed_codec_boost, log_callback=lambda msg: self._log_event("match_log", str(msg)), ) self.match_results = results diff --git a/main_gui.py b/main_gui.py index e9e69f9..99cc76c 100644 --- a/main_gui.py +++ b/main_gui.py @@ -54,6 +54,12 @@ from io import BytesIO from PIL import Image, ImageTk from mutagen import File as MutagenFile +from config import ( + EXACT_DUPLICATE_THRESHOLD, + MIXED_CODEC_THRESHOLD_BOOST, + NEAR_DUPLICATE_THRESHOLD, + load_config, +) from fingerprint_cache import get_fingerprint from simple_duplicate_finder import SUPPORTED_EXTS, _compute_fp from tag_fixer import MIN_INTERACTIVE_SCORE, FileRecord @@ -1873,7 +1879,39 @@ def _generate_plan(self, write_preview: bool) -> None: self._set_status("Idle", progress=0) return self._set_status("Building plan…", progress=25) - plan = build_consolidation_plan(tracks) + cfg = load_config() + exact_threshold = float(cfg.get("exact_duplicate_threshold", EXACT_DUPLICATE_THRESHOLD)) + near_threshold = float(cfg.get("near_duplicate_threshold", NEAR_DUPLICATE_THRESHOLD)) + mixed_codec_boost = float(cfg.get("mixed_codec_threshold_boost", MIXED_CODEC_THRESHOLD_BOOST)) + fingerprint_settings = { + "trim_silence": bool(cfg.get("trim_silence", False)), + "fingerprint_offset_ms": int(cfg.get("fingerprint_offset_ms", 0)), + "fingerprint_duration_ms": int(cfg.get("fingerprint_duration_ms", 0)), + "fingerprint_silence_threshold_db": float(cfg.get("fingerprint_silence_threshold_db", -50.0)), + "fingerprint_silence_min_len_ms": int(cfg.get("fingerprint_silence_min_len_ms", 500)), + "fingerprint_trim_lead_max_ms": int(cfg.get("fingerprint_trim_lead_max_ms", 500)), + "fingerprint_trim_trail_max_ms": int(cfg.get("fingerprint_trim_trail_max_ms", 500)), + "fingerprint_trim_padding_ms": int(cfg.get("fingerprint_trim_padding_ms", 100)), + "allow_mismatched_edits": bool(cfg.get("allow_mismatched_edits", True)), + } + threshold_settings = { + "exact_duplicate_threshold": exact_threshold, + "near_duplicate_threshold": near_threshold, + "mixed_codec_threshold_boost": mixed_codec_boost, + } + self._log_action( + "Duplicate scan thresholds: " + f"exact={exact_threshold:.3f}, near={near_threshold:.3f}, mixed_codec_boost={mixed_codec_boost:.3f}" + ) + self._log_action(f"Fingerprint settings: {fingerprint_settings}") + plan = build_consolidation_plan( + tracks, + exact_duplicate_threshold=exact_threshold, + near_duplicate_threshold=near_threshold, + mixed_codec_threshold_boost=mixed_codec_boost, + fingerprint_settings=fingerprint_settings, + threshold_settings=threshold_settings, + ) self._plan = plan self.group_disposition_overrides.clear() self._apply_deletion_mode() diff --git a/music_indexer_api.py b/music_indexer_api.py index 449e611..0c9f645 100644 --- a/music_indexer_api.py +++ b/music_indexer_api.py @@ -8,7 +8,7 @@ from collections import defaultdict from typing import Dict, List from dry_run_coordinator import DryRunCoordinator -from config import load_config +from config import MIXED_CODEC_THRESHOLD_BOOST, load_config def _verify_dependencies() -> None: """Raise RuntimeError if the real mutagen library is missing.""" @@ -435,12 +435,14 @@ def _compute(path: str) -> tuple[int | None, str | None]: fp = fp.decode("utf-8") except Exception: fp = fp.decode("latin1", errors="ignore") + ext = os.path.splitext(fullpath)[1].lower() file_infos[fullpath] = { "primary": primary.lower(), "title": title.lower(), "album": album.lower(), "fp": fp, "meta_count": meta_count, + "ext": ext, } to_delete: Dict[str, str] = {} @@ -490,6 +492,8 @@ def _compute(path: str) -> tuple[int | None, str | None]: # --- Near-duplicate detection ------------------------------------------------- from near_duplicate_detector import find_near_duplicates log_callback(" • Detecting near-duplicates…") + cfg = load_config() + mixed_codec_boost = float(cfg.get("mixed_codec_threshold_boost", MIXED_CODEC_THRESHOLD_BOOST)) near_dupes = find_near_duplicates( {p: file_infos[p] for p in kept_files}, EXT_PRIORITY, @@ -498,6 +502,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: enable_phase_c, coord, max_workers, + mixed_codec_boost=mixed_codec_boost, ) if near_dupes.review_required: log_callback( diff --git a/near_duplicate_detector.py b/near_duplicate_detector.py index 22d4d4e..3901faf 100644 --- a/near_duplicate_detector.py +++ b/near_duplicate_detector.py @@ -21,6 +21,7 @@ COARSE_FP_BAND_SIZE = 8 COARSE_FP_BANDS = 6 COARSE_FP_QUANTIZATION = 4 +LOSSLESS_EXTS = {".flac", ".wav", ".alac", ".ape", ".aiff", ".aif"} @dataclass @@ -131,6 +132,34 @@ def _metadata_gate(info_a: Mapping[str, str | None], info_b: Mapping[str, str | return False +def _path_ext(path: str, info: Mapping[str, str | None]) -> str: + ext = info.get("ext") if isinstance(info, Mapping) else None + if ext: + return str(ext).lower() if str(ext).startswith(".") else f".{str(ext).lower()}" + return os.path.splitext(path)[1].lower() + + +def _is_lossless(ext: str) -> bool: + return ext in LOSSLESS_EXTS + + +def _pair_threshold( + path_a: str, + path_b: str, + info_a: Mapping[str, str | None], + info_b: Mapping[str, str | None], + base_threshold: float, + mixed_codec_boost: float, +) -> float: + ext_a = _path_ext(path_a, info_a) + ext_b = _path_ext(path_b, info_b) + if not ext_a or not ext_b: + return base_threshold + if _is_lossless(ext_a) != _is_lossless(ext_b): + return base_threshold + mixed_codec_boost + return base_threshold + + def _coarse_fingerprint_keys(fp: str | None) -> List[str]: parsed = _parse_fp(fp) if fp else None values: List[int] = [] @@ -177,6 +206,7 @@ def _scan_paths( paths: Iterable[str], file_infos: Dict[str, Dict[str, str | None]], threshold: float, + mixed_codec_boost: float, log_callback, ) -> Tuple[List[Tuple[Set[str], float]], int, int, int]: """Scan a list of paths for near duplicate clusters.""" @@ -196,12 +226,21 @@ def _scan_paths( coarse_blocked += 1 continue dist = fingerprint_distance(file_infos[p]["fp"], file_infos[q]["fp"]) - if dist <= threshold: + pair_threshold = _pair_threshold( + p, + q, + file_infos[p], + file_infos[q], + threshold, + mixed_codec_boost, + ) + if dist <= pair_threshold: adj[p].add(q) adj[q].add(p) pair_distances[frozenset({p, q})] = dist log_callback( - f" → near-dup {os.path.basename(p)} vs {os.path.basename(q)} dist={dist:.3f}" + f" → near-dup {os.path.basename(p)} vs {os.path.basename(q)} " + f"dist={dist:.3f} thr={pair_threshold:.3f}" ) comparisons += 1 @@ -233,10 +272,17 @@ def _scan_album( paths: Iterable[str], file_infos: Dict[str, Dict[str, str | None]], threshold: float, + mixed_codec_boost: float, log_callback, ) -> tuple[str, List[Tuple[Set[str], float]], int, int, int]: """Worker helper: scan a single album for near duplicates.""" - clusters, comparisons, meta_blocked, coarse_blocked = _scan_paths(paths, file_infos, threshold, log_callback) + clusters, comparisons, meta_blocked, coarse_blocked = _scan_paths( + paths, + file_infos, + threshold, + mixed_codec_boost, + log_callback, + ) return album, clusters, comparisons, meta_blocked, coarse_blocked @@ -248,6 +294,7 @@ def find_near_duplicates( enable_cross_album: bool = False, coord=None, max_workers: int | None = None, + mixed_codec_boost: float = 0.0, ) -> NearDuplicateResult: """Return reviewable near-duplicate groups (auto-merging is gated by review).""" if log_callback is None: @@ -278,7 +325,7 @@ def log_callback(msg: str) -> None: f"Phase B: [{idx}/{total_albums}] Scanning album '{album}' ({len(album_paths)} tracks)" ) html_lines.append(f"Album: {album or ''} ({len(album_paths)} tracks)") - futures[ex.submit(_scan_album, album, album_paths, file_infos, threshold, log_callback)] = album + futures[ex.submit(_scan_album, album, album_paths, file_infos, threshold, mixed_codec_boost, log_callback)] = album for fut in as_completed(futures): album = futures[fut] @@ -321,7 +368,7 @@ def log_callback(msg: str) -> None: f"Phase C: [{idx}/{total_songs}] Scanning '{title}' ({len(song_paths)} tracks)" ) cross_lines.append(f"{title or ''} ({len(song_paths)} tracks)") - futures[ex.submit(_scan_paths, song_paths, file_infos, threshold, log_callback)] = (prim, title) + futures[ex.submit(_scan_paths, song_paths, file_infos, threshold, mixed_codec_boost, log_callback)] = (prim, title) for fut in as_completed(futures): _key = futures[fut] diff --git a/simple_duplicate_finder.py b/simple_duplicate_finder.py index 24f8ed4..cfbe8f5 100644 --- a/simple_duplicate_finder.py +++ b/simple_duplicate_finder.py @@ -7,6 +7,7 @@ from fingerprint_cache import get_fingerprint from near_duplicate_detector import fingerprint_distance import chromaprint_utils +from config import load_config, FP_DURATION_MS, FP_OFFSET_MS, FP_SILENCE_MIN_LEN_MS, FP_SILENCE_THRESHOLD_DB SUPPORTED_EXTS = {".flac", ".m4a", ".aac", ".mp3", ".wav", ".ogg"} EXT_PRIORITY = {".flac": 0, ".m4a": 1, ".aac": 1, ".mp3": 2, ".wav": 3, ".ogg": 4} @@ -36,11 +37,22 @@ def _dlog(label: str, msg: str) -> None: def _compute_fp(path: str) -> Tuple[Optional[int], Optional[str]]: """Compute fingerprint for ``path`` using Chromaprint.""" try: + cfg = load_config() + start_sec = float(cfg.get("fingerprint_offset_ms", FP_OFFSET_MS)) / 1000.0 + duration_sec = float(cfg.get("fingerprint_duration_ms", FP_DURATION_MS)) / 1000.0 + duration_sec = duration_sec if duration_sec > 0 else 120.0 + trim_silence = bool(cfg.get("trim_silence", False)) + silence_threshold_db = float( + cfg.get("fingerprint_silence_threshold_db", FP_SILENCE_THRESHOLD_DB) + ) + silence_min_len_ms = float(cfg.get("fingerprint_silence_min_len_ms", FP_SILENCE_MIN_LEN_MS)) fp = chromaprint_utils.fingerprint_fpcalc( path, - trim=True, - start_sec=5.0, - duration_sec=60.0, + trim=trim_silence, + start_sec=start_sec, + duration_sec=duration_sec, + threshold_db=silence_threshold_db, + min_silence_duration=silence_min_len_ms / 1000.0, ) _dlog("FP", f"computed fingerprint for {path}: {fp}") _dlog("FP", f"prefix={fp[:FP_PREFIX_LEN]}") @@ -227,4 +239,3 @@ def compute(path: str) -> Tuple[Optional[int], Optional[str]]: if progress_callback: progress_callback("complete", len(duplicates), 0, "") return duplicates, missing_fp - diff --git a/tests/test_fingerprint_generator.py b/tests/test_fingerprint_generator.py index 056f9fb..dda7338 100644 --- a/tests/test_fingerprint_generator.py +++ b/tests/test_fingerprint_generator.py @@ -118,7 +118,7 @@ def test_mtime_change_triggers_recompute(monkeypatch, tmp_path): conn.close() def fake_compute(args): - path, _db_path, _trim = args + path, _db_path, _settings = args return path, 9, "new-fp", None # ensure filesystem mtime advances diff --git a/tests/test_library_sync.py b/tests/test_library_sync.py index a1a5b93..9fdb8b4 100644 --- a/tests/test_library_sync.py +++ b/tests/test_library_sync.py @@ -28,7 +28,7 @@ def File(path, easy=False): fingerprint_stub = types.ModuleType('fingerprint_generator') -def fake_compute(paths, db_path, log_callback=None, progress_callback=None, cancel_event=None): +def fake_compute(paths, db_path, log_callback=None, progress_callback=None, cancel_event=None, **_kwargs): for p in paths: yield p, 0, fp_map.get(os.path.basename(p), 'x') @@ -107,7 +107,7 @@ def fake_fp(path, **kw): return "1 1 1 1 1 1 1 2" return "1 1 1 1 1 1 1 1" - def fake_compute(paths, db_path, log_callback=None, progress_callback=None, cancel_event=None): + def fake_compute(paths, db_path, log_callback=None, progress_callback=None, cancel_event=None, **_kwargs): for p in paths: fp = fake_fp(p) yield p, 0, fp @@ -163,7 +163,7 @@ def fake_fp(path, **kw): return "1 2 3 4" return "1 2 3 5" - def fake_compute(paths, db_path, log_callback=None, progress_callback=None, cancel_event=None): + def fake_compute(paths, db_path, log_callback=None, progress_callback=None, cancel_event=None, **_kwargs): for p in paths: fp = fake_fp(p) yield p, 0, fp @@ -211,7 +211,7 @@ def fake_fp(path, **kw): return "10 20 30 40 50" return "10 20 30 40 51" - def fake_compute(paths, db_path, log_callback=None, progress_callback=None, cancel_event=None): + def fake_compute(paths, db_path, log_callback=None, progress_callback=None, cancel_event=None, **_kwargs): for p in paths: fp = fake_fp(p) yield p, 0, fp diff --git a/tests/test_library_sync_integration.py b/tests/test_library_sync_integration.py index 3bd3dea..cb27327 100644 --- a/tests/test_library_sync_integration.py +++ b/tests/test_library_sync_integration.py @@ -25,7 +25,7 @@ def _setup_media_stubs(monkeypatch, fp_map, corrupt=None, missing=None, cancel_a missing = missing or set() fp_mod = _ensure_fp_module() - def fake_compute(paths, db_path, log_callback=None, progress_callback=None, cancel_event=None): + def fake_compute(paths, db_path, log_callback=None, progress_callback=None, cancel_event=None, **_kwargs): for idx, path in enumerate(paths): if cancel_after is not None and idx >= cancel_after: if cancel_event: From bf914e417811c83a78549516b2b94d31f744fd10 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Thu, 1 Jan 2026 11:32:01 -0500 Subject: [PATCH 216/606] Fix HTML section list closure --- duplicate_consolidation.py | 39 +++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index 962b7a4..1fa3e49 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -2481,25 +2481,26 @@ def _format_setting(value: object) -> str: html_lines.extend( [ - "
      ", - "
      ", - "
      ", - "", - "", - "
      ", - "
      ", - "0 visible", - "", - "", - "
      ", - "
      ", - "
      ", - ] + "", + "
      ", + "
      ", + "", + "", + "
      ", + "
      ", + "0 visible", + "", + "", + "
      ", + "
      ", + "
      ", + ] + ) all_actions: List[Dict[str, object]] = [] for group in plan.groups: From f552bf5cd5b73106085a110dbc3de9bd739a5947 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Thu, 1 Jan 2026 11:41:20 -0500 Subject: [PATCH 217/606] Add configurable thresholds UI and report toggle --- duplicate_consolidation.py | 19 +++- main_gui.py | 193 +++++++++++++++++++++++++++++++++++++ 2 files changed, 210 insertions(+), 2 deletions(-) diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index 1fa3e49..13bc63b 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -2246,6 +2246,9 @@ def _format_setting(value: object) -> str: "cursor:pointer;", "}", "button.copy:active{ transform: translateY(1px); }", + ".settings-panel{", + "margin-top: 10px;", + "}", "details.group{", "border:1px solid var(--border);", "border-radius: 10px;", @@ -2466,11 +2469,15 @@ def _format_setting(value: object) -> str: ] if settings_entries: + insert_at = html_lines.index( + "" + ) + 1 + html_lines.insert(insert_at, "") html_lines.extend( [ - "
      ", + "") - bucket_stats = group_pair_stats.get(group.group_id, {}) bucket_diag = group.bucket_diagnostics or {} bucket_sources = bucket_diag.get("sources") if isinstance(bucket_diag.get("sources"), Mapping) else {} bucket_size = len(bucket_sources) if bucket_sources else len(group_paths) @@ -3759,23 +3870,10 @@ def _format_setting(value: object) -> str: isinstance(bucket_sources, Mapping) and any(source == "metadata" for source in bucket_sources.values()) ) - has_artwork = ( - isinstance(bucket_sources, Mapping) - and any(source == "artwork" for source in bucket_sources.values()) - ) or bool(bucket_diag.get("artwork_merges")) - if has_metadata and has_artwork: - formation = "metadata + artwork" - elif has_metadata: + if has_metadata: formation = "metadata-seeded" - elif has_artwork: - formation = "artwork pull-in" else: formation = "fallback" - best_distance = bucket_stats.get("best") - best_distance_label = f"{best_distance:.4f}" if isinstance(best_distance, float) else "n/a" - exact_count = int(bucket_stats.get("exact", 0) or 0) - near_count = int(bucket_stats.get("near", 0) or 0) - no_match_count = int(bucket_stats.get("no_match", 0) or 0) edges = bucket_stats.get("edges", []) if isinstance(bucket_stats.get("edges"), list) else [] no_match_edges = ( bucket_stats.get("no_match_edges", []) diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index a1a5158..a92d4c4 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -1221,9 +1221,13 @@ def _trace_panel(group: object) -> List[str]: lines.append( f"
      {html.escape(_format_trace_value(summary.get('missing_album')))}
      " ) - lines.append("
      Artwork merges
      ") + lines.append("
      Artwork variants
      ") lines.append( - f"
      {html.escape(_format_trace_value(summary.get('artwork_merges')))}
      " + f"
      {html.escape(_format_trace_value(summary.get('artwork_variants')))}
      " + ) + lines.append("
      Artwork unknown
      ") + lines.append( + f"
      {html.escape(_format_trace_value(summary.get('artwork_unknown')))}
      " ) lines.append("
      Thresholds
      ") lines.append( @@ -1333,6 +1337,15 @@ def _trace_panel(group: object) -> List[str]: lines.append( f"
      {html.escape(_format_trace_value(artwork_hashing))}
      " ) + artwork_status = ( + trace_payload.get("artwork_status") + if isinstance(trace_payload.get("artwork_status"), Mapping) + else {} + ) + lines.append("
      Artwork status
      ") + lines.append( + f"
      {html.escape(_format_trace_value(artwork_status))}
      " + ) artwork_comp = ( trace_payload.get("artwork_comparison") if isinstance(trace_payload.get("artwork_comparison"), Mapping) @@ -2053,6 +2066,14 @@ def _format_setting(value: object) -> str: f"Status: {state}. " + "; ".join(badges) if badges else f"Status: {state}." ) group_paths = {winner_path, *getattr(grp, "losers", [])} + bucket_stats = group_pair_stats.get(gid, {}) + best_distance = bucket_stats.get("best") + best_distance_label = ( + f"{best_distance:.4f}" if isinstance(best_distance, float) else "n/a" + ) + exact_count = int(bucket_stats.get("exact", 0) or 0) + near_count = int(bucket_stats.get("near", 0) or 0) + no_match_count = int(bucket_stats.get("no_match", 0) or 0) html_lines.append( "
      str: html_lines.append(f"
      {_path_row(winner_path)}
      ") html_lines.append("
      Group ID
      ") html_lines.append(f"
      {html.escape(gid)}
      ") + html_lines.append("
      Metadata bucket key
      ") + html_lines.append( + f"
      {html.escape(str(getattr(grp, 'grouping_metadata_key', '—')))}
      " + ) + html_lines.append("
      Audio identity
      ") + html_lines.append( + "
      " + f"{html.escape(str(getattr(grp, 'group_match_type', '')))} · best {html.escape(best_distance_label)} " + f"(exact {exact_count} · near {near_count} · no-match {no_match_count})" + "
      " + ) + html_lines.append("
      Artwork variant
      ") + html_lines.append( + f"
      {html.escape(str(getattr(grp, 'artwork_variant_label', '')))}
      " + ) + if getattr(grp, "artwork_variant_total", 1) > 1: + html_lines.append("
      Artwork variants
      ") + html_lines.append( + "
      Audio-identical but different artwork variants preserved.
      " + ) + unknown_reasons = getattr(grp, "artwork_unknown_reasons", {}) or {} + if isinstance(unknown_reasons, Mapping) and unknown_reasons: + missing_notes = ", ".join( + f"{_basename(path)} ({reason})" + for path, reason in sorted(unknown_reasons.items(), key=lambda item: str(item[0]).lower()) + ) + html_lines.append("
      Artwork missing/unreadable
      ") + html_lines.append(f"
      {html.escape(missing_notes)}
      ") html_lines.append("
      Debug
      ") html_lines.append( "
      Shown for traceability; safe to ignore for normal review.
      " @@ -2183,7 +2232,6 @@ def _format_setting(value: object) -> str: html_lines.append("
      ") html_lines.append("
      ") html_lines.append("
      ") - bucket_stats = group_pair_stats.get(gid, {}) bucket_diag = getattr(grp, "bucket_diagnostics", {}) or {} bucket_sources = ( bucket_diag.get("sources") @@ -2196,25 +2244,10 @@ def _format_setting(value: object) -> str: isinstance(bucket_sources, Mapping) and any(source == "metadata" for source in bucket_sources.values()) ) - has_artwork = ( - isinstance(bucket_sources, Mapping) - and any(source == "artwork" for source in bucket_sources.values()) - ) or bool(bucket_diag.get("artwork_merges")) - if has_metadata and has_artwork: - formation = "metadata + artwork" - elif has_metadata: + if has_metadata: formation = "metadata-seeded" - elif has_artwork: - formation = "artwork pull-in" else: formation = "fallback" - best_distance = bucket_stats.get("best") - best_distance_label = ( - f"{best_distance:.4f}" if isinstance(best_distance, float) else "n/a" - ) - exact_count = int(bucket_stats.get("exact", 0) or 0) - near_count = int(bucket_stats.get("near", 0) or 0) - no_match_count = int(bucket_stats.get("no_match", 0) or 0) edges = bucket_stats.get("edges", []) if isinstance(bucket_stats.get("edges"), list) else [] no_match_edges = ( bucket_stats.get("no_match_edges", []) diff --git a/tests/test_duplicate_consolidation_guards.py b/tests/test_duplicate_consolidation_guards.py index 4527e2c..1d48faf 100644 --- a/tests/test_duplicate_consolidation_guards.py +++ b/tests/test_duplicate_consolidation_guards.py @@ -34,6 +34,11 @@ def _make_group(tmp_path, *, review_flags=None, disposition="quarantine"): artwork_candidates=[], chosen_artwork_source={}, artwork_status="unchanged", + artwork_variant_id=1, + artwork_variant_total=1, + artwork_variant_label="Single artwork variant", + artwork_unknown_tracks=[], + artwork_unknown_reasons={}, loser_disposition={str(loser): disposition}, playlist_rewrites={str(loser): str(winner)}, playlist_impact=PlaylistImpact(playlists=1, entries=1), diff --git a/tests/test_duplicate_consolidation_hardening.py b/tests/test_duplicate_consolidation_hardening.py index 4328f2a..5554690 100644 --- a/tests/test_duplicate_consolidation_hardening.py +++ b/tests/test_duplicate_consolidation_hardening.py @@ -118,6 +118,11 @@ def test_playlist_rewrite_validation_dry_run(tmp_path): artwork_candidates=[], chosen_artwork_source={}, artwork_status="unchanged", + artwork_variant_id=1, + artwork_variant_total=1, + artwork_variant_label="Single artwork variant", + artwork_unknown_tracks=[], + artwork_unknown_reasons={}, loser_disposition={str(loser): "quarantine"}, playlist_rewrites={str(loser): str(winner)}, playlist_impact=PlaylistImpact(playlists=1, entries=1), @@ -187,6 +192,11 @@ def test_execute_from_preview_output_path(tmp_path): artwork_candidates=[], chosen_artwork_source={}, artwork_status="unchanged", + artwork_variant_id=1, + artwork_variant_total=1, + artwork_variant_label="Single artwork variant", + artwork_unknown_tracks=[], + artwork_unknown_reasons={}, loser_disposition={str(loser): "quarantine"}, playlist_rewrites={str(loser): str(winner)}, playlist_impact=PlaylistImpact(playlists=1, entries=1), From 4b6792758f326a31928b80ada8cfede4babcd879 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sat, 3 Jan 2026 23:46:35 -0500 Subject: [PATCH 255/606] Fix M4A cover art extraction --- utils/audio_metadata_reader.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/utils/audio_metadata_reader.py b/utils/audio_metadata_reader.py index 2056a03..23c0ed5 100644 --- a/utils/audio_metadata_reader.py +++ b/utils/audio_metadata_reader.py @@ -4,6 +4,7 @@ import importlib.util import logging import os +from collections.abc import Mapping from typing import Dict, Iterable, List, Optional, Tuple from utils.path_helpers import ensure_long_path @@ -194,13 +195,13 @@ def _extract_cover_payloads(audio) -> List[bytes]: payloads.append(data) break - covr = tags.get("covr") if isinstance(tags, dict) else None + covr = tags.get("covr") if isinstance(tags, Mapping) else None if covr: for cover in covr: if cover: payloads.append(bytes(cover)) - wmp = tags.get("WM/Picture") if isinstance(tags, dict) else None + wmp = tags.get("WM/Picture") if isinstance(tags, Mapping) else None if wmp: for pic in wmp: data = getattr(pic, "value", None) @@ -265,6 +266,10 @@ def read_metadata( if error is None: error = f"read failed: {exc}" cover_payloads = _extract_cover_payloads(audio_full) + if not cover_payloads: + sidecar = read_sidecar_artwork_bytes(path) + if sidecar: + cover_payloads = [sidecar] return tags, cover_payloads, error, reader_hint From 4544c029576073e0ee3564094e02e8b3f887b79b Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sat, 3 Jan 2026 23:46:45 -0500 Subject: [PATCH 256/606] Add artwork variant toggle to report --- duplicate_consolidation.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index 4f5df99..823f751 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -3146,6 +3146,20 @@ def _trace_panel(group: GroupPlan) -> List[str]: lines.append("") return lines + def _has_artwork_variants(group: GroupPlan, threshold: float) -> bool: + if getattr(group, "artwork_variant_total", 1) > 1: + return True + winner_hash = group.artwork_hashes.get(group.winner_path) + if winner_hash is None: + return False + for loser in group.losers: + loser_hash = group.artwork_hashes.get(loser) + if loser_hash is None: + continue + if _hamming_distance(winner_hash, loser_hash) > threshold: + return True + return False + generated_at_iso = plan.generated_at.isoformat() host_name = platform.node() or "unknown-host" reports_dir = os.path.dirname(output_html_path) @@ -3673,6 +3687,9 @@ def _format_setting(value: object) -> str: "", "", "", + "", "
      ", "
      ", "0 visible", @@ -3692,6 +3709,7 @@ def _format_setting(value: object) -> str: badges = _action_badges(group, actions) group_disposition_count = _group_disposition_count(group) group_type_hint = _group_type_hint(actions) + group_has_art_variants = _has_artwork_variants(group, art_threshold) search_tokens = [group.group_id, group.winner_path, _basename(group.winner_path), state] search_tokens.extend(badges) for act in actions: @@ -3714,6 +3732,7 @@ def _format_setting(value: object) -> str: "
      str: "const groups = () => $$('#groups details.group');" "const searchEl = $('#search');" "const filterEl = $('#filter');" + "const artToggleEl = $('#toggleArtVariants');" "const visibleCountEl = $('#visibleCount');" "const expandAllBtn = $('#expandAll');" "const collapseAllBtn = $('#collapseAll');" @@ -4100,11 +4120,14 @@ def _format_setting(value: object) -> str: "const hasQuarantine = (d.getAttribute('data-has-quarantine') || 'false') === 'true';" "const hasFailure = (d.getAttribute('data-has-failure') || 'false') === 'true';" "const typeHint = (d.getAttribute('data-type') || '').toLowerCase();" + "const hasArtVariant = (d.getAttribute('data-has-art-variant') || 'false') === 'true';" + "const showArtVariants = artToggleEl ? artToggleEl.checked : true;" "let matchesFilter = true;" "if (mode === 'has-quarantine') matchesFilter = hasQuarantine;" "if (mode === 'metadata-only') matchesFilter = typeHint === 'metadata-only';" "if (mode === 'failed') matchesFilter = hasFailure;" - "const show = matchesSearch && matchesFilter;" + "const matchesArtVariant = showArtVariants || !hasArtVariant;" + "const show = matchesSearch && matchesFilter && matchesArtVariant;" "d.classList.toggle('hidden', !show);" "if (show) visible += 1;" "}" @@ -4161,6 +4184,7 @@ def _format_setting(value: object) -> str: "}" "searchEl?.addEventListener('input', applyFilters);" "filterEl?.addEventListener('change', applyFilters);" + "artToggleEl?.addEventListener('change', applyFilters);" "computeStats();" "applyFilters();" "})();" From f186d3890ff1169b14b23d05be8fc0cef74b8a13 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 4 Jan 2026 09:44:12 -0500 Subject: [PATCH 257/606] Add artwork variant toggle to execution report --- duplicate_consolidation_executor.py | 32 ++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index a92d4c4..3c5576f 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -1510,6 +1510,26 @@ def _run_status() -> tuple[str, str, str]: emoji = "✅" if success else "❌" return status, status_class, f"{emoji} {status}" + def _has_artwork_variants(group: object | None, threshold: float) -> bool: + if group is None: + return False + if getattr(group, "artwork_variant_total", 1) > 1: + return True + winner_path = getattr(group, "winner_path", "") + if not winner_path: + return False + artwork_hashes = getattr(group, "artwork_hashes", {}) or {} + winner_hash = artwork_hashes.get(winner_path) + if winner_hash is None: + return False + for loser in getattr(group, "losers", []) or []: + loser_hash = artwork_hashes.get(loser) + if loser_hash is None: + continue + if _hamming_distance(winner_hash, loser_hash) > threshold: + return True + return False + settings_entries: List[tuple[str, object]] = [] if plan and plan.threshold_settings: settings_entries.extend( @@ -2016,6 +2036,9 @@ def _format_setting(value: object) -> str: "", "", "", + "", "
      ", "
      ", "0 visible", @@ -2054,6 +2077,7 @@ def _format_setting(value: object) -> str: group_type_hint = _group_type_hint(group_actions) group_has_quarantine = _group_has_quarantine(group_actions) group_has_failure = _group_has_failure(group_actions) + group_has_art_variants = _has_artwork_variants(grp, art_threshold) search_tokens = [gid, winner_path, _basename(winner_path), state] search_tokens.extend(badges) for act in group_actions: @@ -2078,6 +2102,7 @@ def _format_setting(value: object) -> str: "
      str: "const groups = () => $$('#groups details.group');" "const searchEl = $('#search');" "const filterEl = $('#filter');" + "const artToggleEl = $('#toggleArtVariants');" "const visibleCountEl = $('#visibleCount');" "const expandAllBtn = $('#expandAll');" "const collapseAllBtn = $('#collapseAll');" @@ -2475,12 +2501,15 @@ def _format_setting(value: object) -> str: "const matchesSearch = !term || hay.includes(term);" "const hasQuarantine = (d.getAttribute('data-has-quarantine') || 'false') === 'true';" "const hasFailure = (d.getAttribute('data-has-failure') || 'false') === 'true';" + "const hasArtVariant = (d.getAttribute('data-has-art-variant') || 'false') === 'true';" + "const showArtVariants = artToggleEl ? artToggleEl.checked : true;" "const typeHint = (d.getAttribute('data-type') || '').toLowerCase();" "let matchesFilter = true;" "if (mode === 'has-quarantine') matchesFilter = hasQuarantine;" "if (mode === 'metadata-only') matchesFilter = typeHint === 'metadata-only';" "if (mode === 'failed') matchesFilter = hasFailure;" - "const show = matchesSearch && matchesFilter;" + "const matchesArtVariant = showArtVariants || !hasArtVariant;" + "const show = matchesSearch && matchesFilter && matchesArtVariant;" "d.classList.toggle('hidden', !show);" "if (show) visible += 1;" "}" @@ -2539,6 +2568,7 @@ def _format_setting(value: object) -> str: "}" "searchEl?.addEventListener('input', applyFilters);" "filterEl?.addEventListener('change', applyFilters);" + "artToggleEl?.addEventListener('change', applyFilters);" "computeStats();" "applyFilters();" "})();" From 04310db0a51a0aa7ac6b6755ddcc86b5109bc817 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 4 Jan 2026 09:51:53 -0500 Subject: [PATCH 258/606] Revert "Add 'Show different artwork variants' toggle to execution report controls" --- duplicate_consolidation_executor.py | 32 +---------------------------- 1 file changed, 1 insertion(+), 31 deletions(-) diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index 3c5576f..a92d4c4 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -1510,26 +1510,6 @@ def _run_status() -> tuple[str, str, str]: emoji = "✅" if success else "❌" return status, status_class, f"{emoji} {status}" - def _has_artwork_variants(group: object | None, threshold: float) -> bool: - if group is None: - return False - if getattr(group, "artwork_variant_total", 1) > 1: - return True - winner_path = getattr(group, "winner_path", "") - if not winner_path: - return False - artwork_hashes = getattr(group, "artwork_hashes", {}) or {} - winner_hash = artwork_hashes.get(winner_path) - if winner_hash is None: - return False - for loser in getattr(group, "losers", []) or []: - loser_hash = artwork_hashes.get(loser) - if loser_hash is None: - continue - if _hamming_distance(winner_hash, loser_hash) > threshold: - return True - return False - settings_entries: List[tuple[str, object]] = [] if plan and plan.threshold_settings: settings_entries.extend( @@ -2036,9 +2016,6 @@ def _format_setting(value: object) -> str: "", "", "", - "", "
      ", "
      ", "0 visible", @@ -2077,7 +2054,6 @@ def _format_setting(value: object) -> str: group_type_hint = _group_type_hint(group_actions) group_has_quarantine = _group_has_quarantine(group_actions) group_has_failure = _group_has_failure(group_actions) - group_has_art_variants = _has_artwork_variants(grp, art_threshold) search_tokens = [gid, winner_path, _basename(winner_path), state] search_tokens.extend(badges) for act in group_actions: @@ -2102,7 +2078,6 @@ def _format_setting(value: object) -> str: "
      str: "const groups = () => $$('#groups details.group');" "const searchEl = $('#search');" "const filterEl = $('#filter');" - "const artToggleEl = $('#toggleArtVariants');" "const visibleCountEl = $('#visibleCount');" "const expandAllBtn = $('#expandAll');" "const collapseAllBtn = $('#collapseAll');" @@ -2501,15 +2475,12 @@ def _format_setting(value: object) -> str: "const matchesSearch = !term || hay.includes(term);" "const hasQuarantine = (d.getAttribute('data-has-quarantine') || 'false') === 'true';" "const hasFailure = (d.getAttribute('data-has-failure') || 'false') === 'true';" - "const hasArtVariant = (d.getAttribute('data-has-art-variant') || 'false') === 'true';" - "const showArtVariants = artToggleEl ? artToggleEl.checked : true;" "const typeHint = (d.getAttribute('data-type') || '').toLowerCase();" "let matchesFilter = true;" "if (mode === 'has-quarantine') matchesFilter = hasQuarantine;" "if (mode === 'metadata-only') matchesFilter = typeHint === 'metadata-only';" "if (mode === 'failed') matchesFilter = hasFailure;" - "const matchesArtVariant = showArtVariants || !hasArtVariant;" - "const show = matchesSearch && matchesFilter && matchesArtVariant;" + "const show = matchesSearch && matchesFilter;" "d.classList.toggle('hidden', !show);" "if (show) visible += 1;" "}" @@ -2568,7 +2539,6 @@ def _format_setting(value: object) -> str: "}" "searchEl?.addEventListener('input', applyFilters);" "filterEl?.addEventListener('change', applyFilters);" - "artToggleEl?.addEventListener('change', applyFilters);" "computeStats();" "applyFilters();" "})();" From b631028be0546bf86e5db57c6603dfea5a6213e1 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 4 Jan 2026 10:04:24 -0500 Subject: [PATCH 259/606] Fix M4A tester artwork and persist last folder --- main_gui.py | 79 +++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 70 insertions(+), 9 deletions(-) diff --git a/main_gui.py b/main_gui.py index bd04f2f..b7f93d7 100644 --- a/main_gui.py +++ b/main_gui.py @@ -5,6 +5,7 @@ import webbrowser import re import html +import shutil from pathlib import Path if sys.platform == "win32": @@ -114,7 +115,11 @@ from controllers.cluster_controller import gather_tracks from playlist_generator import write_playlist from utils.path_helpers import ensure_long_path, strip_ext_prefix -from utils.audio_metadata_reader import read_metadata, read_tags +from utils.audio_metadata_reader import ( + read_metadata, + read_tags, + read_sidecar_artwork_bytes, +) FilterFn = Callable[[FileRecord], bool] _cached_filters = None @@ -877,9 +882,11 @@ def open_path(path: str) -> None: messagebox.showerror("Open File", f"Could not open {path}: {exc}") def browse(): - f = filedialog.askopenfilename() + initial_dir = load_last_path() or os.getcwd() + f = filedialog.askopenfilename(initialdir=initial_dir) if f: sel.set(f) + save_last_path(os.path.dirname(f)) def append_log(msg: str) -> None: def _append() -> None: @@ -1906,19 +1913,29 @@ def _validate_library_root(self) -> str | None: return path def _browse_library(self) -> None: - chosen = filedialog.askdirectory(title="Select Library Root") + initial_dir = self.library_path_var.get().strip() or load_last_path() or os.getcwd() + chosen = filedialog.askdirectory( + title="Select Library Root", + initialdir=initial_dir, + ) if not chosen: return self.library_path_var.set(chosen) self.playlist_path_var.set(self._default_playlist_folder(chosen)) self._log_action(f"Library path updated to {chosen}") + save_last_path(chosen) def _browse_playlist(self) -> None: - chosen = filedialog.askdirectory(title="Select Playlist Folder") + initial_dir = self.playlist_path_var.get().strip() or load_last_path() or os.getcwd() + chosen = filedialog.askdirectory( + title="Select Playlist Folder", + initialdir=initial_dir, + ) if not chosen: return self.playlist_path_var.set(chosen) self._log_action(f"Playlist folder updated to {chosen}") + save_last_path(chosen) def _open_threshold_settings(self) -> None: dlg = tk.Toplevel(self) @@ -2632,9 +2649,14 @@ def _toggle_advanced(self) -> None: self.advanced_visible = True def _choose_file(self, var: tk.StringVar) -> None: - path = filedialog.askopenfilename(title="Select audio file") + initial_dir = load_last_path() or os.getcwd() + path = filedialog.askopenfilename( + title="Select audio file", + initialdir=initial_dir, + ) if path: var.set(path) + save_last_path(os.path.dirname(path)) def _parse_float(self, raw: str, label: str, *, min_value: float | None = None) -> float | None: try: @@ -3117,10 +3139,11 @@ def __init__(self, parent: tk.Widget): ttk.Button(controls, text="Close", command=self.destroy).pack(side="left") def _choose_file(self) -> None: + initial_dir = load_last_path() or os.getcwd() chosen = filedialog.askopenfilename( parent=self, title="Select M4A File", - initialdir=os.getcwd(), + initialdir=initial_dir, filetypes=[("M4A files", "*.m4a")], ) if not chosen: @@ -3129,13 +3152,14 @@ def _choose_file(self) -> None: messagebox.showerror("M4A Tester", "Please select a .m4a file.") return self.file_path_var.set(chosen) + save_last_path(os.path.dirname(chosen)) self._load_metadata(chosen) def _load_metadata(self, path: str) -> None: self._set_metadata_text("") self._set_cover_image(None) - tags, cover_payloads, error, _reader = read_metadata(path, include_cover=True) + tags, _cover_payloads, error, _reader = read_metadata(path, include_cover=False) def _format_value(value: object) -> str | None: if value in (None, "", []): @@ -3167,7 +3191,7 @@ def _format_value(value: object) -> str | None: lines.append(f"Metadata error: {error}") self._set_metadata_text("\n".join(lines) if lines else "No metadata found.") - cover_bytes = cover_payloads[0] if cover_payloads else None + cover_bytes = self._extract_cover_art_bytes(path) if cover_bytes: try: image = Image.open(BytesIO(cover_bytes)) @@ -3178,6 +3202,38 @@ def _format_value(value: object) -> str | None: else: self.cover_label.config(text="No album art found") + def _extract_cover_art_bytes(self, path: str) -> bytes | None: + if shutil.which("ffmpeg"): + cmd = [ + "ffmpeg", + "-v", + "error", + "-i", + ensure_long_path(path), + "-map", + "0:v", + "-frames:v", + "1", + "-f", + "image2pipe", + "-vcodec", + "png", + "-", + ] + try: + result = subprocess.run( + cmd, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + except Exception: + result = None + if result and result.returncode == 0 and result.stdout: + return result.stdout + + return read_sidecar_artwork_bytes(path) + def _set_metadata_text(self, content: str) -> None: self.metadata_text.configure(state="normal") self.metadata_text.delete("1.0", tk.END) @@ -3235,9 +3291,14 @@ def __init__(self, parent: tk.Widget): ttk.Label(container, textvariable=self.status_var).pack(anchor="w", pady=(8, 0)) def _browse_folder(self) -> None: - chosen = filedialog.askdirectory(title="Select Folder for Duplicate Bucketing") + initial_dir = load_last_path() or os.getcwd() + chosen = filedialog.askdirectory( + title="Select Folder for Duplicate Bucketing", + initialdir=initial_dir, + ) if chosen: self.folder_var.set(chosen) + save_last_path(chosen) def _set_running(self, running: bool, status: str) -> None: self.status_var.set(status) From d8e8ef54e25f568102b265ae7cd164ea81249fff Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 4 Jan 2026 10:14:35 -0500 Subject: [PATCH 260/606] Add Opus tester dialog --- main_gui.py | 146 +++++++++++++++++++++++++++ utils/opus_metadata_reader.py | 180 ++++++++++++++++++++++++++++++++++ 2 files changed, 326 insertions(+) create mode 100644 utils/opus_metadata_reader.py diff --git a/main_gui.py b/main_gui.py index b7f93d7..70b34f5 100644 --- a/main_gui.py +++ b/main_gui.py @@ -120,6 +120,7 @@ read_tags, read_sidecar_artwork_bytes, ) +from utils.opus_metadata_reader import read_opus_metadata FilterFn = Callable[[FileRecord], bool] _cached_filters = None @@ -3249,6 +3250,147 @@ def _set_cover_image(self, image: Image.Image | None) -> None: self.cover_label.configure(image=self._cover_photo, text="") +class OpusTesterDialog(tk.Toplevel): + """Standalone UI for validating Opus metadata/album art parsing.""" + + def __init__(self, parent: tk.Widget): + super().__init__(parent) + self.title("Opus Tester") + self.transient(parent) + self.resizable(True, True) + self.parent = parent + + self.file_path_var = tk.StringVar() + self._cover_photo: ImageTk.PhotoImage | None = None + + container = ttk.Frame(self, padding=12) + container.pack(fill="both", expand=True) + + ttk.Label( + container, + text="Select an Opus file to validate album art and metadata parsing.", + foreground="#555", + wraplength=540, + ).pack(anchor="w") + + file_frame = ttk.LabelFrame(container, text="Opus File") + file_frame.pack(fill="x", pady=(10, 8)) + ttk.Label(file_frame, text="File").grid( + row=0, column=0, sticky="w", padx=6, pady=6 + ) + entry = ttk.Entry( + file_frame, textvariable=self.file_path_var, width=60, state="readonly" + ) + entry.grid(row=0, column=1, sticky="ew", padx=6, pady=6) + ttk.Button(file_frame, text="Browse…", command=self._choose_file).grid( + row=0, column=2, sticky="e", padx=6, pady=6 + ) + file_frame.columnconfigure(1, weight=1) + + content = ttk.Frame(container) + content.pack(fill="both", expand=True) + + art_frame = ttk.LabelFrame(content, text="Album Art") + art_frame.pack(side="left", fill="both", expand=False, padx=(0, 8)) + self.cover_label = ttk.Label( + art_frame, + text="No album art loaded", + anchor="center", + width=28, + padding=8, + ) + self.cover_label.pack(fill="both", expand=True) + + metadata_frame = ttk.LabelFrame(content, text="Metadata") + metadata_frame.pack(side="left", fill="both", expand=True) + self.metadata_text = ScrolledText( + metadata_frame, height=16, wrap="word", state="disabled" + ) + self.metadata_text.pack(fill="both", expand=True, padx=6, pady=6) + + controls = ttk.Frame(container) + controls.pack(fill="x", pady=(8, 0)) + ttk.Button(controls, text="Close", command=self.destroy).pack(side="left") + + def _choose_file(self) -> None: + initial_dir = load_last_path() or os.getcwd() + chosen = filedialog.askopenfilename( + parent=self, + title="Select Opus File", + initialdir=initial_dir, + filetypes=[("Opus files", "*.opus")], + ) + if not chosen: + return + if not chosen.lower().endswith(".opus"): + messagebox.showerror("Opus Tester", "Please select a .opus file.") + return + self.file_path_var.set(chosen) + save_last_path(os.path.dirname(chosen)) + self._load_metadata(chosen) + + def _load_metadata(self, path: str) -> None: + self._set_metadata_text("") + self._set_cover_image(None) + + tags, cover_payloads, error = read_opus_metadata(path) + + def _format_value(value: object) -> str | None: + if value in (None, "", []): + return None + if isinstance(value, (list, tuple)): + return ", ".join(str(item) for item in value if item not in (None, "")) + return str(value) + + lines = [] + track_value = _format_value(tags.get("tracknumber") or tags.get("track")) + disc_value = _format_value(tags.get("discnumber") or tags.get("disc")) + display_fields = [ + ("Title", "title"), + ("Artist", "artist"), + ("Album", "album"), + ("Album Artist", "albumartist"), + ("Track", track_value), + ("Disc", disc_value), + ("Year", "year"), + ("Date", "date"), + ("Genre", "genre"), + ("Compilation", "compilation"), + ] + for label, key in display_fields: + value = _format_value(tags.get(key)) if isinstance(key, str) else key + if value: + lines.append(f"{label}: {value}") + if error and not lines: + lines.append(f"Metadata error: {error}") + self._set_metadata_text("\n".join(lines) if lines else "No metadata found.") + + cover_bytes = cover_payloads[0] if cover_payloads else None + if cover_bytes: + try: + image = Image.open(BytesIO(cover_bytes)) + image.thumbnail((240, 240)) + self._set_cover_image(image) + except Exception: + self.cover_label.config(text="Failed to load album art") + else: + self.cover_label.config(text="No album art found") + + def _set_metadata_text(self, content: str) -> None: + self.metadata_text.configure(state="normal") + self.metadata_text.delete("1.0", tk.END) + self.metadata_text.insert("1.0", content) + self.metadata_text.configure(state="disabled") + + def _set_cover_image(self, image: Image.Image | None) -> None: + if image is None: + self._cover_photo = None + self.cover_label.configure(image="", text="No album art loaded") + return + self._cover_photo = ImageTk.PhotoImage(image) + self.cover_label.configure(image=self._cover_photo, text="") + + class DuplicateBucketingPocDialog(tk.Toplevel): """Minimal UI for the Duplicate Bucketing proof-of-concept tool.""" @@ -3517,6 +3659,7 @@ def build_ui(self): command=self._open_duplicate_bucketing_poc_tool, ) tools_menu.add_command(label="M4A Tester…", command=self._open_m4a_tester_tool) + tools_menu.add_command(label="Opus Tester…", command=self._open_opus_tester_tool) tools_menu.add_separator() tools_menu.add_command(label="Reset Tag-Fix Log", command=self.reset_tagfix_log) menubar.add_cascade(label="Tools", menu=tools_menu) @@ -5659,6 +5802,9 @@ def _open_duplicate_bucketing_poc_tool(self) -> None: def _open_m4a_tester_tool(self) -> None: M4ATesterDialog(self) + def _open_opus_tester_tool(self) -> None: + OpusTesterDialog(self) + def _create_player_library_table(self, parent: tk.Widget) -> ttk.Frame: table = ttk.Frame(parent) p_vsb = ttk.Scrollbar(table, orient="vertical") diff --git a/utils/opus_metadata_reader.py b/utils/opus_metadata_reader.py new file mode 100644 index 0000000..64a91ef --- /dev/null +++ b/utils/opus_metadata_reader.py @@ -0,0 +1,180 @@ +"""Helpers for reading Opus metadata and embedded cover art.""" +from __future__ import annotations + +import base64 +import importlib.util +import logging +import os +from typing import Dict, List, Optional, Tuple + +from utils.path_helpers import ensure_long_path + +logger = logging.getLogger(__name__) + +_MUTAGEN_AVAILABLE = importlib.util.find_spec("mutagen") is not None +if _MUTAGEN_AVAILABLE: + from mutagen.flac import Picture # type: ignore + from mutagen.oggopus import OggOpus # type: ignore +else: # pragma: no cover - optional dependency + Picture = None # type: ignore + OggOpus = None # type: ignore + + +TAG_KEYS = ( + "artist", + "albumartist", + "title", + "album", + "date", + "year", + "track", + "tracknumber", + "disc", + "discnumber", + "genre", + "compilation", +) + +SIDECAR_ARTWORK_SUFFIXES = (".artwork", ".cover", ".jpg") + + +def _blank_tags() -> Dict[str, object]: + return {key: None for key in TAG_KEYS} + + +def _first_value(value: object) -> object: + if isinstance(value, list): + return _first_value(value[0]) if value else None + if isinstance(value, tuple): + return _first_value(value[0]) if value else None + if isinstance(value, bytes): + for encoding in ("utf-8", "utf-16", "latin-1"): + try: + return value.decode(encoding) + except UnicodeDecodeError: + continue + return value.decode("utf-8", errors="replace") + return value + + +def _parse_int(value: object) -> int | None: + try: + return int(str(value).split("/")[0]) + except Exception: + return None + + +def _parse_year(value: object) -> int | None: + if value in (None, ""): + return None + text = str(value) + if len(text) >= 4 and text[:4].isdigit(): + return int(text[:4]) + digits = "".join(ch for ch in text if ch.isdigit()) + if len(digits) >= 4: + return int(digits[:4]) + return None + + +def _read_sidecar_artwork_bytes(path: str) -> bytes | None: + for suffix in SIDECAR_ARTWORK_SUFFIXES: + candidate = f"{path}{suffix}" + if os.path.exists(candidate): + try: + with open(candidate, "rb") as handle: + return handle.read() + except OSError: + continue + return None + + +def _get_tag(tags: Dict[str, object], key: str) -> object | None: + lowered = key.lower() + for tag_key, tag_value in tags.items(): + if tag_key.lower() == lowered: + return tag_value + return None + + +def _decode_picture_payload(payload: str) -> bytes | None: + if Picture is None: + return None + try: + raw = base64.b64decode(payload) + picture = Picture(raw) + return picture.data or None + except Exception as exc: + logger.debug("Failed to decode Opus picture payload: %s", exc) + return None + + +def _decode_coverart_payload(payload: str) -> bytes | None: + try: + return base64.b64decode(payload) + except Exception as exc: + logger.debug("Failed to decode Opus coverart payload: %s", exc) + return None + + +def _extract_cover_payloads(tags: Dict[str, object]) -> List[bytes]: + payloads: List[bytes] = [] + + block_pictures = _get_tag(tags, "metadata_block_picture") + if block_pictures: + for item in block_pictures if isinstance(block_pictures, list) else [block_pictures]: + data = _decode_picture_payload(str(item)) + if data: + payloads.append(data) + + coverart = _get_tag(tags, "coverart") + if coverart: + for item in coverart if isinstance(coverart, list) else [coverart]: + data = _decode_coverart_payload(str(item)) + if data: + payloads.append(data) + + return payloads + + +def read_opus_metadata( + path: str, +) -> Tuple[Dict[str, object], List[bytes], Optional[str]]: + """Return (tags, cover_payloads, error) for an Opus file.""" + tags = _blank_tags() + cover_payloads: List[bytes] = [] + error: Optional[str] = None + + if OggOpus is None: + return tags, cover_payloads, "mutagen unavailable" + + try: + audio = OggOpus(ensure_long_path(path)) + except Exception as exc: # pragma: no cover - depends on local files + return tags, cover_payloads, f"read failed: {exc}" + + raw_tags = audio.tags or {} + + tags["title"] = _first_value(_get_tag(raw_tags, "title")) + tags["artist"] = _first_value(_get_tag(raw_tags, "artist")) + tags["album"] = _first_value(_get_tag(raw_tags, "album")) + tags["albumartist"] = _first_value(_get_tag(raw_tags, "albumartist")) + tags["date"] = _first_value(_get_tag(raw_tags, "date") or _get_tag(raw_tags, "year")) + tags["genre"] = _first_value(_get_tag(raw_tags, "genre")) + tags["compilation"] = _first_value(_get_tag(raw_tags, "compilation")) + + track_raw = _first_value(_get_tag(raw_tags, "tracknumber") or _get_tag(raw_tags, "track")) + disc_raw = _first_value(_get_tag(raw_tags, "discnumber") or _get_tag(raw_tags, "disc")) + tags["tracknumber"] = track_raw + tags["discnumber"] = disc_raw + tags["track"] = _parse_int(track_raw) if track_raw not in (None, "") else None + tags["disc"] = _parse_int(disc_raw) if disc_raw not in (None, "") else None + + tags["year"] = _parse_year(tags.get("date")) + + cover_payloads = _extract_cover_payloads(raw_tags) + if not cover_payloads: + sidecar = _read_sidecar_artwork_bytes(path) + if sidecar: + cover_payloads = [sidecar] + + return tags, cover_payloads, error From 754cd0b3ca3fb659f1138764a04f3ab248acb00d Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 4 Jan 2026 10:33:33 -0500 Subject: [PATCH 261/606] Add artwork variant report filter setting --- config.py | 2 + duplicate_consolidation.py | 70 ++++++++++++++--------------- duplicate_consolidation_executor.py | 35 ++++++++++++++- main_gui.py | 31 ++++++++++++- 4 files changed, 99 insertions(+), 39 deletions(-) diff --git a/config.py b/config.py index 3e4cf31..a991ab3 100644 --- a/config.py +++ b/config.py @@ -79,6 +79,7 @@ def load_config(): cfg.setdefault("allow_mismatched_edits", ALLOW_MISMATCHED_EDITS) cfg.setdefault("mixed_codec_threshold_boost", MIXED_CODEC_THRESHOLD_BOOST) cfg.setdefault("artwork_vastly_different_threshold", ARTWORK_VASTLY_DIFFERENT_THRESHOLD) + cfg.setdefault("duplicate_finder_show_artwork_variants", True) cfg.setdefault("library_root", "") cfg.setdefault("use_library_sync_review", False) cfg.setdefault("library_sync_review", {}) @@ -103,6 +104,7 @@ def load_config(): "library_sync_review": {}, "mixed_codec_threshold_boost": MIXED_CODEC_THRESHOLD_BOOST, "artwork_vastly_different_threshold": ARTWORK_VASTLY_DIFFERENT_THRESHOLD, + "duplicate_finder_show_artwork_variants": True, } diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index 823f751..6351b94 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -2793,7 +2793,12 @@ def _track_row(track: DuplicateTrack, label: str) -> List[str]: return output_html_path -def export_consolidation_preview_html(plan: ConsolidationPlan, output_html_path: str) -> str: +def export_consolidation_preview_html( + plan: ConsolidationPlan, + output_html_path: str, + *, + show_artwork_variants: bool = True, +) -> str: """Write an HTML preview of the consolidation plan.""" plan.refresh_plan_signature() @@ -2887,7 +2892,7 @@ def _action_badges(group: GroupPlan, actions: List[Dict[str, object]]) -> List[s meta_count = sum(1 for act in actions if act["step"] == "metadata") art_count = sum(1 for act in actions if act["step"] == "artwork") playlist_count = sum(1 for act in actions if act["step"] == "playlist") - loser_count = len(group.losers) + loser_count = sum(1 for act in actions if act["step"] == "loser_cleanup") if meta_count: badges.append(f"metadata updates ({meta_count})") if art_count: @@ -2905,12 +2910,12 @@ def _group_type_hint(actions: List[Dict[str, object]]) -> str: return "metadata-only" return "mixed" - def _group_disposition_count(group: GroupPlan) -> int: - return sum( - 1 - for loser in group.losers - if group.loser_disposition.get(loser, "quarantine") != "retain" - ) + def _is_artwork_variant(group: GroupPlan, loser_path: str, threshold: float) -> bool: + winner_hash = group.artwork_hashes.get(group.winner_path) + loser_hash = group.artwork_hashes.get(loser_path) + if winner_hash is None or loser_hash is None: + return False + return _hamming_distance(winner_hash, loser_hash) > threshold def _album_art_src( group: GroupPlan, @@ -3146,20 +3151,6 @@ def _trace_panel(group: GroupPlan) -> List[str]: lines.append("
      ") return lines - def _has_artwork_variants(group: GroupPlan, threshold: float) -> bool: - if getattr(group, "artwork_variant_total", 1) > 1: - return True - winner_hash = group.artwork_hashes.get(group.winner_path) - if winner_hash is None: - return False - for loser in group.losers: - loser_hash = group.artwork_hashes.get(loser) - if loser_hash is None: - continue - if _hamming_distance(winner_hash, loser_hash) > threshold: - return True - return False - generated_at_iso = plan.generated_at.isoformat() host_name = platform.node() or "unknown-host" reports_dir = os.path.dirname(output_html_path) @@ -3687,9 +3678,6 @@ def _format_setting(value: object) -> str: "", "", "", - "", "
      ", "
      ", "0 visible", @@ -3704,12 +3692,30 @@ def _format_setting(value: object) -> str: all_actions: List[Dict[str, object]] = [] for group in plan.groups: actions = _planned_actions(group) + visible_losers = list(group.losers) + hidden_losers: set[str] = set() + if not show_artwork_variants: + hidden_losers = { + loser for loser in group.losers if _is_artwork_variant(group, loser, art_threshold) + } + if hidden_losers: + visible_losers = [loser for loser in group.losers if loser not in hidden_losers] + actions = [ + act + for act in actions + if not (act["step"] == "loser_cleanup" and act["target"] in hidden_losers) + ] + if not show_artwork_variants and not actions and not visible_losers: + continue all_actions.extend(actions) state = "review" if group.review_flags else "ready" badges = _action_badges(group, actions) - group_disposition_count = _group_disposition_count(group) + group_disposition_count = sum( + 1 + for loser in visible_losers + if group.loser_disposition.get(loser, "quarantine") != "retain" + ) group_type_hint = _group_type_hint(actions) - group_has_art_variants = _has_artwork_variants(group, art_threshold) search_tokens = [group.group_id, group.winner_path, _basename(group.winner_path), state] search_tokens.extend(badges) for act in actions: @@ -3732,7 +3738,6 @@ def _format_setting(value: object) -> str: "
      str: for act in actions if act["step"] == "loser_cleanup" } - loser_paths = group.losers or [""] + loser_paths = visible_losers winner_art_src = _album_art_src(group, group.winner_path) winner_art_hash = group.artwork_hashes.get(group.winner_path) html_lines.append("
      ") @@ -4076,7 +4081,6 @@ def _format_setting(value: object) -> str: "const groups = () => $$('#groups details.group');" "const searchEl = $('#search');" "const filterEl = $('#filter');" - "const artToggleEl = $('#toggleArtVariants');" "const visibleCountEl = $('#visibleCount');" "const expandAllBtn = $('#expandAll');" "const collapseAllBtn = $('#collapseAll');" @@ -4120,14 +4124,11 @@ def _format_setting(value: object) -> str: "const hasQuarantine = (d.getAttribute('data-has-quarantine') || 'false') === 'true';" "const hasFailure = (d.getAttribute('data-has-failure') || 'false') === 'true';" "const typeHint = (d.getAttribute('data-type') || '').toLowerCase();" - "const hasArtVariant = (d.getAttribute('data-has-art-variant') || 'false') === 'true';" - "const showArtVariants = artToggleEl ? artToggleEl.checked : true;" "let matchesFilter = true;" "if (mode === 'has-quarantine') matchesFilter = hasQuarantine;" "if (mode === 'metadata-only') matchesFilter = typeHint === 'metadata-only';" "if (mode === 'failed') matchesFilter = hasFailure;" - "const matchesArtVariant = showArtVariants || !hasArtVariant;" - "const show = matchesSearch && matchesFilter && matchesArtVariant;" + "const show = matchesSearch && matchesFilter;" "d.classList.toggle('hidden', !show);" "if (show) visible += 1;" "}" @@ -4184,7 +4185,6 @@ def _format_setting(value: object) -> str: "}" "searchEl?.addEventListener('input', applyFilters);" "filterEl?.addEventListener('change', applyFilters);" - "artToggleEl?.addEventListener('change', applyFilters);" "computeStats();" "applyFilters();" "})();" diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index a92d4c4..9e89b6e 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -82,6 +82,7 @@ class ExecutionConfig: dry_run_execute: bool = False retain_losers: bool = False quarantine_flatten: bool = False + show_artwork_variants: bool = True @dataclass @@ -1525,6 +1526,7 @@ def _format_setting(value: object) -> str: return f"{value:.3f}" return str(value) + show_artwork_variants = config.show_artwork_variants group_pair_stats = { group.group_id: _pairwise_stats(group) for group in (plan.groups if plan else []) } @@ -1536,6 +1538,15 @@ def _format_setting(value: object) -> str: ) ) + def _is_artwork_variant(group: object, loser_path: str, threshold: float) -> bool: + artwork_hashes = getattr(group, "artwork_hashes", {}) or {} + winner_path = getattr(group, "winner_path", "") + winner_hash = artwork_hashes.get(winner_path) + loser_hash = artwork_hashes.get(loser_path) + if winner_hash is None or loser_hash is None: + return False + return _hamming_distance(winner_hash, loser_hash) > threshold + html_lines = [ "", "", @@ -2048,6 +2059,24 @@ def _format_setting(value: object) -> str: grp = group_lookup.get(gid) winner_path = getattr(grp, "winner_path", "Unknown winner") group_actions = actions_by_group.get(gid, []) + visible_losers = list(getattr(grp, "losers", []) or []) + if not show_artwork_variants and grp: + hidden_losers = { + loser + for loser in getattr(grp, "losers", []) + if _is_artwork_variant(grp, loser, art_threshold) + } + if hidden_losers: + visible_losers = [ + loser for loser in getattr(grp, "losers", []) if loser not in hidden_losers + ] + group_actions = [ + act + for act in group_actions + if not (act.step == "loser_cleanup" and act.target in hidden_losers) + ] + if not show_artwork_variants and not group_actions and not visible_losers: + continue state = _group_state(group_actions) badges = _action_badges(group_actions) group_quarantine_count = _group_quarantine_count(group_actions) @@ -2065,7 +2094,7 @@ def _format_setting(value: object) -> str: summary_line = ( f"Status: {state}. " + "; ".join(badges) if badges else f"Status: {state}." ) - group_paths = {winner_path, *getattr(grp, "losers", [])} + group_paths = {winner_path, *visible_losers} bucket_stats = group_pair_stats.get(gid, {}) best_distance = bucket_stats.get("best") best_distance_label = ( @@ -2167,8 +2196,10 @@ def _format_setting(value: object) -> str: html_lines.append("
      ") if cleanup_actions: loser_entries = cleanup_actions - else: + elif show_artwork_variants or visible_losers: loser_entries = [None] + else: + loser_entries = [] for loser_action in loser_entries: loser_path = loser_action.target if loser_action else "" loser_disposition = "" diff --git a/main_gui.py b/main_gui.py index 70b34f5..60b3efb 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1505,6 +1505,10 @@ def __init__(self, parent: tk.Widget, library_path: str): self.quarantine_var = tk.BooleanVar(value=True) self.delete_losers_var = tk.BooleanVar(value=False) self.override_review_var = tk.BooleanVar(value=False) + cfg = load_config() + self.show_artwork_variants_var = tk.BooleanVar( + value=cfg.get("duplicate_finder_show_artwork_variants", True) + ) self.group_disposition_var = tk.StringVar(value="") self.group_disposition_overrides: dict[str, str] = {} self._selected_group_id: str | None = None @@ -1574,6 +1578,12 @@ def __init__(self, parent: tk.Widget, library_path: str): ttk.Button(controls, text="⚙️ Thresholds", command=self._open_threshold_settings).pack( side="left", padx=(0, 12) ) + ttk.Checkbutton( + controls, + text="Show different artwork variants", + variable=self.show_artwork_variants_var, + command=self._toggle_show_artwork_variants, + ).pack(side="left", padx=(0, 10)) ttk.Checkbutton( controls, text="Update Playlists", @@ -2225,7 +2235,11 @@ def _generate_plan(self, write_preview: bool) -> None: self.preview_json_path = json_path self._log_action(f"Audit JSON written to {json_path}") html_path = os.path.join(docs_dir, "duplicate_preview.html") - export_consolidation_preview_html(plan, html_path) + export_consolidation_preview_html( + plan, + html_path, + show_artwork_variants=self.show_artwork_variants_var.get(), + ) self.preview_html_path = html_path self._log_action(f"Preview HTML written to {html_path}") self._set_status("Preview generated", progress=100) @@ -2307,7 +2321,11 @@ def _handle_execute(self) -> None: preview_html_path = os.path.join(path, "Docs", "duplicate_preview.html") plan_for_html = plan_for_checks or self._load_preview_plan(preview_plan_path) if plan_for_html: - export_consolidation_preview_html(plan_for_html, preview_html_path) + export_consolidation_preview_html( + plan_for_html, + preview_html_path, + show_artwork_variants=self.show_artwork_variants_var.get(), + ) self.preview_html_path = preview_html_path if not plan_for_checks: plan_for_checks = self._load_preview_plan(preview_plan_path) @@ -2394,6 +2412,7 @@ def log_callback(msg: str) -> None: retain_losers=not self.quarantine_var.get(), allow_deletion=deletions_requested, confirm_deletion=deletions_requested, + show_artwork_variants=self.show_artwork_variants_var.get(), ) if not self.quarantine_var.get(): @@ -2458,6 +2477,14 @@ def _toggle_update_playlists(self) -> None: state = "enabled" if self.update_playlists_var.get() else "disabled" self._log_action(f"Update Playlists {state}") + def _toggle_show_artwork_variants(self) -> None: + enabled = self.show_artwork_variants_var.get() + cfg = load_config() + cfg["duplicate_finder_show_artwork_variants"] = bool(enabled) + save_config(cfg) + state = "enabled" if enabled else "disabled" + self._log_action(f"Show different artwork variants {state}") + def _toggle_quarantine(self) -> None: state = "enabled" if self.quarantine_var.get() else "disabled" self._log_action(f"Quarantine Duplicates {state}") From a10c9716742b5fb54d525245583748d89f66bf95 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 4 Jan 2026 10:44:02 -0500 Subject: [PATCH 262/606] Fix sanitize for non-string tags --- music_indexer_api.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/music_indexer_api.py b/music_indexer_api.py index 9d49ca4..0754f9c 100644 --- a/music_indexer_api.py +++ b/music_indexer_api.py @@ -88,9 +88,10 @@ def progress_callback(_c, _t, _m, _p): return counts # ─── Shared Utility Functions ───────────────────────────────────────── -def sanitize(name: str) -> str: +def sanitize(name: object) -> str: invalid = r'<>:"/\\|?*' - return "".join(c for c in (name or "Unknown") if c not in invalid).strip() or "Unknown" + text = "Unknown" if name is None else str(name) + return "".join(c for c in text if c not in invalid).strip() or "Unknown" def collapse_repeats(name: str) -> str: """ From db9394fda493d6d4d44b05b3fdc03a8546948e9f Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 4 Jan 2026 10:52:27 -0500 Subject: [PATCH 263/606] Improve indexer popup and dry run report UX --- main_gui.py | 9 ++++++++- unsorted_popup.py | 13 +++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/main_gui.py b/main_gui.py index 60b3efb..b34b57c 100644 --- a/main_gui.py +++ b/main_gui.py @@ -4442,10 +4442,17 @@ def show_popup(): ) self.after(0, lambda: self.log.insert("end", "✔ Indexing complete\n")) if dry_run: + self.after( + 0, + lambda: webbrowser.open( + Path(output_html).resolve().as_uri() + ), + ) self.after( 0, lambda: messagebox.showinfo( - "Dry Run Complete", f"Preview written to:\n{output_html}" + "Dry Run Complete", + f"Preview opened in your browser:\n{output_html}", ), ) else: diff --git a/unsorted_popup.py b/unsorted_popup.py index 875077e..e64ff26 100644 --- a/unsorted_popup.py +++ b/unsorted_popup.py @@ -31,6 +31,19 @@ def __init__(self, parent: tk.Misc, folder: str): close_btn.pack(side="right") self.protocol("WM_DELETE_WINDOW", self.destroy) + self.wait_visibility() + self._center_on_screen() + self.grab_set() + self.focus_set() + self.bell() + + def _center_on_screen(self) -> None: + self.update_idletasks() + width = self.winfo_reqwidth() + height = self.winfo_reqheight() + x = (self.winfo_screenwidth() // 2) - (width // 2) + y = (self.winfo_screenheight() // 2) - (height // 2) + self.geometry(f"{width}x{height}+{x}+{y}") def open_folder(self) -> None: """Open the Not Sorted folder in the OS file browser.""" From 09ed8453f0eefcbc5f376e0f06836d34b9fb6c51 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 4 Jan 2026 11:03:48 -0500 Subject: [PATCH 264/606] Store tool artifacts in Docs --- controllers/cluster_controller.py | 4 +++- controllers/library_index_controller.py | 4 +++- controllers/normalize_controller.py | 8 +++++--- controllers/tagfix_controller.py | 2 +- main_gui.py | 10 +++++----- update_genres.py | 4 +++- 6 files changed, 20 insertions(+), 12 deletions(-) diff --git a/controllers/cluster_controller.py b/controllers/cluster_controller.py index 1251056..ecfabe6 100644 --- a/controllers/cluster_controller.py +++ b/controllers/cluster_controller.py @@ -55,7 +55,9 @@ def cluster_library( """Generate clustered playlists for ``library_path`` and return features.""" tracks = gather_tracks(library_path, folder_filter) - log_path = os.path.join(library_path, f"{method}_log.txt") + docs_dir = os.path.join(library_path, "Docs") + os.makedirs(docs_dir, exist_ok=True) + log_path = os.path.join(docs_dir, f"{method}_log.txt") def log(msg: str) -> None: log_callback(msg) diff --git a/controllers/library_index_controller.py b/controllers/library_index_controller.py index bb402bb..0e1cd99 100644 --- a/controllers/library_index_controller.py +++ b/controllers/library_index_controller.py @@ -52,6 +52,8 @@ def extract_tags(path: str) -> dict: def generate_index(folder_path: str) -> str: """Generate library_index.html for ``folder_path`` and return its path.""" + docs_dir = os.path.join(folder_path, "Docs") + os.makedirs(docs_dir, exist_ok=True) entries = [] for dirpath, _, files in os.walk(folder_path): for fname in files: @@ -93,7 +95,7 @@ def generate_index(folder_path: str) -> str: html_lines.append("") html_lines.append("") - out_path = os.path.join(folder_path, "library_index.html") + out_path = os.path.join(docs_dir, "library_index.html") with open(out_path, "w", encoding="utf-8") as f: f.write("\n".join(html_lines)) messagebox.showinfo( diff --git a/controllers/normalize_controller.py b/controllers/normalize_controller.py index adc71e0..a428f1a 100644 --- a/controllers/normalize_controller.py +++ b/controllers/normalize_controller.py @@ -27,7 +27,8 @@ def load_mapping(folder: str) -> tuple[Dict[str, str], str]: """Return genre mapping dict and path.""" - path = os.path.join(folder, ".genre_mapping.json") + docs_dir = os.path.join(folder, "Docs") + path = os.path.join(docs_dir, ".genre_mapping.json") mapping: Dict[str, str] = {} if os.path.isfile(path): try: @@ -40,8 +41,9 @@ def load_mapping(folder: str) -> tuple[Dict[str, str], str]: def save_mapping(folder: str, mapping: Dict[str, str]) -> str: """Save mapping JSON and return path.""" - path = os.path.join(folder, ".genre_mapping.json") - os.makedirs(folder, exist_ok=True) + docs_dir = os.path.join(folder, "Docs") + path = os.path.join(docs_dir, ".genre_mapping.json") + os.makedirs(docs_dir, exist_ok=True) with open(path, "w", encoding="utf-8") as f: json.dump(mapping, f, indent=2) return path diff --git a/controllers/tagfix_controller.py b/controllers/tagfix_controller.py index 86d49e6..f40c017 100644 --- a/controllers/tagfix_controller.py +++ b/controllers/tagfix_controller.py @@ -17,7 +17,7 @@ def prepare_library(folder: str) -> tuple[str, dict]: db_path = os.path.join(docs_dir, ".soundvault.db") init_db(db_path) - mapping_path = os.path.join(folder, ".genre_mapping.json") + mapping_path = os.path.join(docs_dir, ".genre_mapping.json") mapping = {} if os.path.isfile(mapping_path): try: diff --git a/main_gui.py b/main_gui.py index b34b57c..f5b04c3 100644 --- a/main_gui.py +++ b/main_gui.py @@ -268,7 +268,7 @@ def km_func(X, p): def update_controls(): lib_var.set(app.library_path or "No library selected") norm_status.set( - os.path.join(app.library_path, ".genre_mapping.json") + os.path.join(app.library_path, "Docs", ".genre_mapping.json") if app.library_path else "Select a library to enable normalization." ) @@ -283,7 +283,7 @@ def update_controls(): intro, text=( "Normalize genre labels across your library by generating or " - "updating the .genre_mapping.json file." + "updating the Docs/.genre_mapping.json file." ), wraplength=480, justify="left", @@ -405,7 +405,7 @@ def finish_scan(folder: str): scan_status.set(f"Found {len(getattr(app, 'raw_genre_list', []))} genres.") populate_raw_genres(getattr(app, "raw_genre_list", [])) prog["value"] = prog["maximum"] - app.mapping_path = os.path.join(folder, ".genre_mapping.json") + app.mapping_path = os.path.join(folder, "Docs", ".genre_mapping.json") def start_scan(): folder = app.require_library() @@ -4217,7 +4217,7 @@ def select_library(self): self.library_path = info["path"] self.library_name_var.set(info["name"]) self.library_path_var.set(info["path"]) - self.mapping_path = os.path.join(self.library_path, ".genre_mapping.json") + self.mapping_path = os.path.join(self.library_path, "Docs", ".genre_mapping.json") self._load_genre_mapping() if hasattr(self, "player_search_var"): self.player_search_var.set("") @@ -4954,7 +4954,7 @@ def fix_tags_gui(self): self.tagfix_folder_var.set(folder) self.tagfix_db_path, _ = prepare_library(folder) - self.mapping_path = os.path.join(folder, ".genre_mapping.json") + self.mapping_path = os.path.join(folder, "Docs", ".genre_mapping.json") self._load_genre_mapping() files = discover_files(folder) diff --git a/update_genres.py b/update_genres.py index 951b6b0..fba1c1e 100644 --- a/update_genres.py +++ b/update_genres.py @@ -213,7 +213,9 @@ def main(): print("No folder selected; exiting.") sys.exit(0) - logpath = os.path.join(folder, "genre_update_log.txt") + docs_dir = os.path.join(folder, "Docs") + os.makedirs(docs_dir, exist_ok=True) + logpath = os.path.join(docs_dir, "genre_update_log.txt") try: logfile = open(logpath, "w", encoding="utf-8") except Exception as e: From e302c046a2f8017ce670a1ab57e36acb89cd446a Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 4 Jan 2026 11:05:40 -0500 Subject: [PATCH 265/606] Handle broken fingerprint workers --- .../indexer_engine/fingerprint_generator.py | 38 +++++++++++++++++-- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/library_sync_indexer_engine/indexer_engine/fingerprint_generator.py b/library_sync_indexer_engine/indexer_engine/fingerprint_generator.py index 06e4387..200de39 100644 --- a/library_sync_indexer_engine/indexer_engine/fingerprint_generator.py +++ b/library_sync_indexer_engine/indexer_engine/fingerprint_generator.py @@ -14,7 +14,7 @@ def _with_thresh(func, *args, silence_threshold_db: float = SILENCE_THRESH, **kw return func(*args, silence_threshold=silence_threshold_db, **kwargs) except TypeError: return func(*args, silence_thresh=silence_threshold_db, **kwargs) -from concurrent.futures import ProcessPoolExecutor +from concurrent.futures import BrokenProcessPool, ProcessPoolExecutor import acoustid from utils.path_helpers import ensure_long_path import audio_norm @@ -230,16 +230,46 @@ def progress_callback(current: int, total: int, msg: str, _phase: str) -> None: if pending: with ProcessPoolExecutor(max_workers=max_workers) as exe: futures = [exe.submit(compute_fingerprint_for_file, item) for item in work_items] - for fut in futures: + for index, (fut, item) in enumerate(zip(futures, work_items)): if cancel_event.is_set(): cancelled = 0 for remaining in futures: if not remaining.done() and remaining.cancel(): cancelled += 1 - log_callback(f"Cancellation requested; attempting to stop after current file ({cancelled} cancelled)") + log_callback( + f"Cancellation requested; attempting to stop after current file ({cancelled} cancelled)" + ) break - path, duration, fp_hash, err = fut.result() + try: + path, duration, fp_hash, err = fut.result() + except BrokenProcessPool as exc: + log_callback(f" ! Fingerprinting workers stopped unexpectedly: {exc}") + log_callback(" • Falling back to sequential fingerprinting for remaining files.") + remaining_items = work_items[index:] + for remaining_item in remaining_items: + if cancel_event.is_set(): + log_callback("Cancellation requested; stopping after current fingerprint") + break + path, duration, fp_hash, err = compute_fingerprint_for_file(remaining_item) + completed += 1 + if err: + log_callback(f" ! Failed fingerprint {path}: {err}") + progress_callback(completed, total, path, phase) + if cancel_event.is_set(): + break + continue + mtime = os.path.getmtime(ensure_long_path(path)) + conn.execute( + "INSERT OR REPLACE INTO fingerprints (path, mtime, duration, fingerprint) VALUES (?, ?, ?, ?)", + (path, mtime, duration, fp_hash), + ) + results.append((path, duration, fp_hash)) + log_callback(f"Fingerprinted {path}") + progress_callback(completed, total, path, phase) + if completed % 50 == 0 or completed == total: + log_callback(f" • Fingerprinting {completed}/{total}") + break completed += 1 if err: log_callback(f" ! Failed fingerprint {path}: {err}") From 1809d6e614c0509de43ef7ce9d41592eabd632a1 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 4 Jan 2026 11:07:59 -0500 Subject: [PATCH 266/606] Revert "Handle BrokenProcessPool and fall back to sequential fingerprinting" --- .../indexer_engine/fingerprint_generator.py | 38 ++----------------- 1 file changed, 4 insertions(+), 34 deletions(-) diff --git a/library_sync_indexer_engine/indexer_engine/fingerprint_generator.py b/library_sync_indexer_engine/indexer_engine/fingerprint_generator.py index 200de39..06e4387 100644 --- a/library_sync_indexer_engine/indexer_engine/fingerprint_generator.py +++ b/library_sync_indexer_engine/indexer_engine/fingerprint_generator.py @@ -14,7 +14,7 @@ def _with_thresh(func, *args, silence_threshold_db: float = SILENCE_THRESH, **kw return func(*args, silence_threshold=silence_threshold_db, **kwargs) except TypeError: return func(*args, silence_thresh=silence_threshold_db, **kwargs) -from concurrent.futures import BrokenProcessPool, ProcessPoolExecutor +from concurrent.futures import ProcessPoolExecutor import acoustid from utils.path_helpers import ensure_long_path import audio_norm @@ -230,46 +230,16 @@ def progress_callback(current: int, total: int, msg: str, _phase: str) -> None: if pending: with ProcessPoolExecutor(max_workers=max_workers) as exe: futures = [exe.submit(compute_fingerprint_for_file, item) for item in work_items] - for index, (fut, item) in enumerate(zip(futures, work_items)): + for fut in futures: if cancel_event.is_set(): cancelled = 0 for remaining in futures: if not remaining.done() and remaining.cancel(): cancelled += 1 - log_callback( - f"Cancellation requested; attempting to stop after current file ({cancelled} cancelled)" - ) + log_callback(f"Cancellation requested; attempting to stop after current file ({cancelled} cancelled)") break - try: - path, duration, fp_hash, err = fut.result() - except BrokenProcessPool as exc: - log_callback(f" ! Fingerprinting workers stopped unexpectedly: {exc}") - log_callback(" • Falling back to sequential fingerprinting for remaining files.") - remaining_items = work_items[index:] - for remaining_item in remaining_items: - if cancel_event.is_set(): - log_callback("Cancellation requested; stopping after current fingerprint") - break - path, duration, fp_hash, err = compute_fingerprint_for_file(remaining_item) - completed += 1 - if err: - log_callback(f" ! Failed fingerprint {path}: {err}") - progress_callback(completed, total, path, phase) - if cancel_event.is_set(): - break - continue - mtime = os.path.getmtime(ensure_long_path(path)) - conn.execute( - "INSERT OR REPLACE INTO fingerprints (path, mtime, duration, fingerprint) VALUES (?, ?, ?, ?)", - (path, mtime, duration, fp_hash), - ) - results.append((path, duration, fp_hash)) - log_callback(f"Fingerprinted {path}") - progress_callback(completed, total, path, phase) - if completed % 50 == 0 or completed == total: - log_callback(f" • Fingerprinting {completed}/{total}") - break + path, duration, fp_hash, err = fut.result() completed += 1 if err: log_callback(f" ! Failed fingerprint {path}: {err}") From 0bdadc6b5b36469d30208f48cde9cff293af4a87 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 4 Jan 2026 11:09:09 -0500 Subject: [PATCH 267/606] Add missing artwork threshold config defaults --- library_sync_indexer_engine/indexer_engine/config.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/library_sync_indexer_engine/indexer_engine/config.py b/library_sync_indexer_engine/indexer_engine/config.py index 736e50b..d95b901 100644 --- a/library_sync_indexer_engine/indexer_engine/config.py +++ b/library_sync_indexer_engine/indexer_engine/config.py @@ -16,6 +16,7 @@ NEAR_DUPLICATE_THRESHOLD = 0.1 EXACT_DUPLICATE_THRESHOLD = 0.02 MIXED_CODEC_THRESHOLD_BOOST = 0.03 +ARTWORK_VASTLY_DIFFERENT_THRESHOLD = 24 # File format quality priority used during Library Sync FORMAT_PRIORITY = {".flac": 3, ".wav": 2, ".mp3": 1} @@ -77,6 +78,11 @@ def load_config(): cfg.setdefault("fingerprint_trim_padding_ms", FP_TRIM_PADDING_MS) cfg.setdefault("allow_mismatched_edits", ALLOW_MISMATCHED_EDITS) cfg.setdefault("mixed_codec_threshold_boost", MIXED_CODEC_THRESHOLD_BOOST) + cfg.setdefault( + "artwork_vastly_different_threshold", + ARTWORK_VASTLY_DIFFERENT_THRESHOLD, + ) + cfg.setdefault("duplicate_finder_show_artwork_variants", True) cfg.setdefault("library_root", "") cfg.setdefault("use_library_sync_review", False) cfg.setdefault("library_sync_review", {}) @@ -100,6 +106,8 @@ def load_config(): "use_library_sync_review": False, "library_sync_review": {}, "mixed_codec_threshold_boost": MIXED_CODEC_THRESHOLD_BOOST, + "artwork_vastly_different_threshold": ARTWORK_VASTLY_DIFFERENT_THRESHOLD, + "duplicate_finder_show_artwork_variants": True, } From 751d94aca026cdbfdcee7d1d152bd6edce9f6260 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 4 Jan 2026 11:16:29 -0500 Subject: [PATCH 268/606] Fix BrokenProcessPool import --- .../indexer_engine/fingerprint_generator.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/library_sync_indexer_engine/indexer_engine/fingerprint_generator.py b/library_sync_indexer_engine/indexer_engine/fingerprint_generator.py index 200de39..ebd9704 100644 --- a/library_sync_indexer_engine/indexer_engine/fingerprint_generator.py +++ b/library_sync_indexer_engine/indexer_engine/fingerprint_generator.py @@ -2,9 +2,16 @@ import sqlite3 import tempfile import threading +from concurrent.futures import ProcessPoolExecutor +from concurrent.futures.process import BrokenProcessPool from typing import Callable, Iterable, List, Tuple + from pydub import AudioSegment, silence +import acoustid +from utils.path_helpers import ensure_long_path +import audio_norm +SUPPORTED_EXTS = {".flac", ".m4a", ".aac", ".mp3", ".wav", ".ogg"} SILENCE_THRESH = -50 # dBFS used for silence detection @@ -14,13 +21,6 @@ def _with_thresh(func, *args, silence_threshold_db: float = SILENCE_THRESH, **kw return func(*args, silence_threshold=silence_threshold_db, **kwargs) except TypeError: return func(*args, silence_thresh=silence_threshold_db, **kwargs) -from concurrent.futures import BrokenProcessPool, ProcessPoolExecutor -import acoustid -from utils.path_helpers import ensure_long_path -import audio_norm - -SUPPORTED_EXTS = {".flac", ".m4a", ".aac", ".mp3", ".wav", ".ogg"} - def _trim_silence(path: str, *, silence_threshold_db: float, min_silence_len_ms: int) -> str: """Return path to temporary file with leading/trailing silence removed.""" From ff527cd8829a5ca298fd2e0011aedafc0762a519 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 4 Jan 2026 13:28:36 -0500 Subject: [PATCH 269/606] Revert "Fix sanitize for non-string tag values" --- music_indexer_api.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/music_indexer_api.py b/music_indexer_api.py index 0754f9c..9d49ca4 100644 --- a/music_indexer_api.py +++ b/music_indexer_api.py @@ -88,10 +88,9 @@ def progress_callback(_c, _t, _m, _p): return counts # ─── Shared Utility Functions ───────────────────────────────────────── -def sanitize(name: object) -> str: +def sanitize(name: str) -> str: invalid = r'<>:"/\\|?*' - text = "Unknown" if name is None else str(name) - return "".join(c for c in text if c not in invalid).strip() or "Unknown" + return "".join(c for c in (name or "Unknown") if c not in invalid).strip() or "Unknown" def collapse_repeats(name: str) -> str: """ From b64fe2d503475062b61dfcc454d72569670f6093 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 4 Jan 2026 13:30:54 -0500 Subject: [PATCH 270/606] Revert "Fix BrokenProcessPool import in fingerprint generator" --- .../indexer_engine/fingerprint_generator.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/library_sync_indexer_engine/indexer_engine/fingerprint_generator.py b/library_sync_indexer_engine/indexer_engine/fingerprint_generator.py index ebd9704..200de39 100644 --- a/library_sync_indexer_engine/indexer_engine/fingerprint_generator.py +++ b/library_sync_indexer_engine/indexer_engine/fingerprint_generator.py @@ -2,16 +2,9 @@ import sqlite3 import tempfile import threading -from concurrent.futures import ProcessPoolExecutor -from concurrent.futures.process import BrokenProcessPool from typing import Callable, Iterable, List, Tuple - from pydub import AudioSegment, silence -import acoustid -from utils.path_helpers import ensure_long_path -import audio_norm -SUPPORTED_EXTS = {".flac", ".m4a", ".aac", ".mp3", ".wav", ".ogg"} SILENCE_THRESH = -50 # dBFS used for silence detection @@ -21,6 +14,13 @@ def _with_thresh(func, *args, silence_threshold_db: float = SILENCE_THRESH, **kw return func(*args, silence_threshold=silence_threshold_db, **kwargs) except TypeError: return func(*args, silence_thresh=silence_threshold_db, **kwargs) +from concurrent.futures import BrokenProcessPool, ProcessPoolExecutor +import acoustid +from utils.path_helpers import ensure_long_path +import audio_norm + +SUPPORTED_EXTS = {".flac", ".m4a", ".aac", ".mp3", ".wav", ".ogg"} + def _trim_silence(path: str, *, silence_threshold_db: float, min_silence_len_ms: int) -> str: """Return path to temporary file with leading/trailing silence removed.""" From 6c7726955961885a99aac0560eb69fa01e96437e Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 4 Jan 2026 13:32:20 -0500 Subject: [PATCH 271/606] Revert "Add missing artwork threshold defaults in indexer engine config" --- library_sync_indexer_engine/indexer_engine/config.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/library_sync_indexer_engine/indexer_engine/config.py b/library_sync_indexer_engine/indexer_engine/config.py index d95b901..736e50b 100644 --- a/library_sync_indexer_engine/indexer_engine/config.py +++ b/library_sync_indexer_engine/indexer_engine/config.py @@ -16,7 +16,6 @@ NEAR_DUPLICATE_THRESHOLD = 0.1 EXACT_DUPLICATE_THRESHOLD = 0.02 MIXED_CODEC_THRESHOLD_BOOST = 0.03 -ARTWORK_VASTLY_DIFFERENT_THRESHOLD = 24 # File format quality priority used during Library Sync FORMAT_PRIORITY = {".flac": 3, ".wav": 2, ".mp3": 1} @@ -78,11 +77,6 @@ def load_config(): cfg.setdefault("fingerprint_trim_padding_ms", FP_TRIM_PADDING_MS) cfg.setdefault("allow_mismatched_edits", ALLOW_MISMATCHED_EDITS) cfg.setdefault("mixed_codec_threshold_boost", MIXED_CODEC_THRESHOLD_BOOST) - cfg.setdefault( - "artwork_vastly_different_threshold", - ARTWORK_VASTLY_DIFFERENT_THRESHOLD, - ) - cfg.setdefault("duplicate_finder_show_artwork_variants", True) cfg.setdefault("library_root", "") cfg.setdefault("use_library_sync_review", False) cfg.setdefault("library_sync_review", {}) @@ -106,8 +100,6 @@ def load_config(): "use_library_sync_review": False, "library_sync_review": {}, "mixed_codec_threshold_boost": MIXED_CODEC_THRESHOLD_BOOST, - "artwork_vastly_different_threshold": ARTWORK_VASTLY_DIFFERENT_THRESHOLD, - "duplicate_finder_show_artwork_variants": True, } From e9471a904c88fcc540e239f2c656c445670cf561 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 4 Jan 2026 13:37:07 -0500 Subject: [PATCH 272/606] Improve indexer apply pipeline cancellation and options --- .../indexer_engine/music_indexer_api.py | 25 +++++++++++++++---- music_indexer_api.py | 25 +++++++++++++++---- 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py index f919a15..c50d29c 100644 --- a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py +++ b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py @@ -31,7 +31,7 @@ class ID3NoHeaderError(Exception): """Fallback error when Mutagen is unavailable.""" pass from utils.audio_metadata_reader import read_tags -from indexer_control import check_cancelled +from indexer_control import check_cancelled, cancel_event # ─── CONFIGURATION ───────────────────────────────────────────────────── COMMON_ARTIST_THRESHOLD = 10 @@ -981,6 +981,9 @@ def apply_indexer_moves( log_callback=None, progress_callback=None, create_playlists: bool = True, + enable_phase_c: bool = False, + flush_cache: bool = False, + max_workers: int | None = None, ): """ 1) Call compute_moves_and_tag_index() to get (moves, tag_index, decision_log). @@ -998,6 +1001,7 @@ def log_callback(msg): def progress_callback(current, total, path, phase): pass + check_cancelled() cfg = load_config() trim_silence = bool(cfg.get("trim_silence", False)) @@ -1021,9 +1025,10 @@ def progress_callback(current, total, path, phase): db_path, log_callback, progress_callback, - None, + max_workers, trim_silence, phase="A", + cancel_event=cancel_event, ) moves, _, _ = compute_moves_and_tag_index( @@ -1031,15 +1036,16 @@ def progress_callback(current, total, path, phase): log_callback, progress_callback, dry_run=False, - enable_phase_c=False, - flush_cache=False, - max_workers=None, + enable_phase_c=enable_phase_c, + flush_cache=flush_cache, + max_workers=max_workers, coord=None, ) total_moves = len(moves) + check_cancelled() MUSIC_ROOT = os.path.join(root_path, "Music") \ if os.path.isdir(os.path.join(root_path, "Music")) \ @@ -1058,6 +1064,7 @@ def progress_callback(current, total, path, phase): # Phase 5: Move audio files for idx, (old_path, new_path) in enumerate(moves.items(), start=1): + check_cancelled() if progress_callback: progress_callback(idx, total_moves, old_path, "C") if idx % 50 == 0 or idx == total_moves: @@ -1079,12 +1086,14 @@ def progress_callback(current, total, path, phase): os.makedirs(trash_dir, exist_ok=True) for dirpath, dirnames, filenames in os.walk(MUSIC_ROOT, topdown=False): + check_cancelled() # 1) Don’t recurse back into Trash/, Docs/, Playlists, or Not Sorted/ dirnames[:] = [d for d in dirnames if d.lower() not in ("trash", "docs", "not sorted", "playlists")] if {"not sorted", "playlists"} & {p.lower() for p in os.path.relpath(dirpath, root_path).split(os.sep)}: continue for fname in filenames: + check_cancelled() full = os.path.join(dirpath, fname) ext = os.path.splitext(fname)[1].lower() @@ -1106,9 +1115,11 @@ def progress_callback(current, total, path, phase): skip_names = {"trash", "docs", "not sorted", "playlists"} touched_dirs = sorted(olddir_to_newdirs.keys(), key=lambda p: p.count(os.sep), reverse=True) for base in touched_dirs: + check_cancelled() if not os.path.isdir(base): continue for dirpath, dirnames, filenames in os.walk(base, topdown=False): + check_cancelled() rel = os.path.relpath(dirpath, root_path) first = rel.split(os.sep, 1)[0].lower() if rel != "." else "" if first in skip_names: @@ -1124,6 +1135,7 @@ def progress_callback(current, total, path, phase): # Phase 6c: Second pass to remove any remaining empty directories for dirpath, dirnames, filenames in os.walk(root_path, topdown=False): + check_cancelled() dirnames[:] = [d for d in dirnames if d.lower() not in skip_names] if os.path.normpath(dirpath) in (root_path, MUSIC_ROOT): continue @@ -1240,6 +1252,9 @@ def log_callback(msg): pass log_callback, progress_callback, create_playlists=create_playlists, + enable_phase_c=enable_phase_c, + flush_cache=flush_cache, + max_workers=max_workers, ) summary = {"moved": actual_summary["moved"], "html": output_html_path, "dry_run": False} return summary diff --git a/music_indexer_api.py b/music_indexer_api.py index 9d49ca4..4b51cf9 100644 --- a/music_indexer_api.py +++ b/music_indexer_api.py @@ -26,7 +26,7 @@ def _verify_dependencies() -> None: def MutagenFile(*_a, **_k): return None from utils.audio_metadata_reader import read_metadata, read_tags -from indexer_control import check_cancelled +from indexer_control import check_cancelled, cancel_event # ─── CONFIGURATION ───────────────────────────────────────────────────── COMMON_ARTIST_THRESHOLD = 10 @@ -965,6 +965,9 @@ def apply_indexer_moves( log_callback=None, progress_callback=None, create_playlists: bool = True, + enable_phase_c: bool = False, + flush_cache: bool = False, + max_workers: int | None = None, ): """ 1) Call compute_moves_and_tag_index() to get (moves, tag_index, decision_log). @@ -982,6 +985,7 @@ def log_callback(msg): def progress_callback(current, total, path, phase): pass + check_cancelled() cfg = load_config() trim_silence = bool(cfg.get("trim_silence", False)) @@ -1005,9 +1009,10 @@ def progress_callback(current, total, path, phase): db_path, log_callback, progress_callback, - None, + max_workers, trim_silence, phase="A", + cancel_event=cancel_event, ) moves, _, _ = compute_moves_and_tag_index( @@ -1015,15 +1020,16 @@ def progress_callback(current, total, path, phase): log_callback, progress_callback, dry_run=False, - enable_phase_c=False, - flush_cache=False, - max_workers=None, + enable_phase_c=enable_phase_c, + flush_cache=flush_cache, + max_workers=max_workers, coord=None, ) total_moves = len(moves) + check_cancelled() MUSIC_ROOT = os.path.join(root_path, "Music") \ if os.path.isdir(os.path.join(root_path, "Music")) \ @@ -1042,6 +1048,7 @@ def progress_callback(current, total, path, phase): # Phase 5: Move audio files for idx, (old_path, new_path) in enumerate(moves.items(), start=1): + check_cancelled() if progress_callback: progress_callback(idx, total_moves, old_path, "C") if idx % 50 == 0 or idx == total_moves: @@ -1063,12 +1070,14 @@ def progress_callback(current, total, path, phase): os.makedirs(trash_dir, exist_ok=True) for dirpath, dirnames, filenames in os.walk(MUSIC_ROOT, topdown=False): + check_cancelled() # 1) Don’t recurse back into Trash/, Docs/, Playlists, or Not Sorted/ dirnames[:] = [d for d in dirnames if d.lower() not in ("trash", "docs", "not sorted", "playlists")] if {"not sorted", "playlists"} & {p.lower() for p in os.path.relpath(dirpath, root_path).split(os.sep)}: continue for fname in filenames: + check_cancelled() full = os.path.join(dirpath, fname) ext = os.path.splitext(fname)[1].lower() @@ -1090,9 +1099,11 @@ def progress_callback(current, total, path, phase): skip_names = {"trash", "docs", "not sorted", "playlists"} touched_dirs = sorted(olddir_to_newdirs.keys(), key=lambda p: p.count(os.sep), reverse=True) for base in touched_dirs: + check_cancelled() if not os.path.isdir(base): continue for dirpath, dirnames, filenames in os.walk(base, topdown=False): + check_cancelled() rel = os.path.relpath(dirpath, root_path) first = rel.split(os.sep, 1)[0].lower() if rel != "." else "" if first in skip_names: @@ -1108,6 +1119,7 @@ def progress_callback(current, total, path, phase): # Phase 6c: Second pass to remove any remaining empty directories for dirpath, dirnames, filenames in os.walk(root_path, topdown=False): + check_cancelled() dirnames[:] = [d for d in dirnames if d.lower() not in skip_names] if os.path.normpath(dirpath) in (root_path, MUSIC_ROOT): continue @@ -1224,6 +1236,9 @@ def log_callback(msg): pass log_callback, progress_callback, create_playlists=create_playlists, + enable_phase_c=enable_phase_c, + flush_cache=flush_cache, + max_workers=max_workers, ) summary = {"moved": actual_summary["moved"], "html": output_html_path, "dry_run": False} return summary From 7bb91ceaf05ed222371d6e34b967cddd146973b0 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 4 Jan 2026 14:28:04 -0500 Subject: [PATCH 273/606] Update docs to reflect current features --- README.md | 20 +++-- docs/library_sync_redesign.md | 140 ++++++++++++++------------------ docs/project_documentation.html | 8 +- 3 files changed, 79 insertions(+), 89 deletions(-) diff --git a/README.md b/README.md index 42eb49b..b060b53 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ SoundVault organizes large music libraries. It deduplicates tracks, fixes tags v git clone --recurse-submodules https://example.com/yourrepo.git ``` - **FFmpeg** installed and on your `PATH` -- **llama.exe** command line for LLM features +- **Optional LLM helper**: `third_party/llama/llama-run.exe` (Windows binaries included) plus a GGUF model (place it at `models/your-model.gguf` or update `plugins/assistant_plugin.py`). ## Installation @@ -62,12 +62,11 @@ python main_gui.py 5. **Clustered Playlists** (interactive K-Means/HDBSCAN) via the Tools ▸ Clustered Playlists menu 6. **Smart Playlist Engine** with tempo/energy buckets 7. **Auto‑DJ** mode builds seamless playlists starting from any song -8. **Tidal-dl Sync** can upgrade low-quality files to FLAC -9. **Library Duplicate Scan** finds duplicate tracks after you drop new songs directly into your library -10. **Cross-Album Scan** optionally finds duplicates appearing on multiple albums -11. Use the **Theme** dropdown and **Help** tab for assistance -12. Launch the **Duplicate Finder** tab to open the updated shell for spotting duplicates -13. Use **Tools → Similarity Inspector** to compare two files and see fingerprint distance details +8. Use the **Library Sync** tab to compare an incoming folder against your existing library, then build/preview/execute an optional copy/move plan. +9. Launch the **Duplicate Finder** tab to open the updated shell for spotting duplicates. +10. **Cross-Album Scan** optionally finds duplicates appearing on multiple albums (Indexer tab → Phase 3). +11. Use the **Theme** dropdown and **Help** tab for assistance. +12. Use **Tools → Similarity Inspector** to compare two files and see fingerprint distance details. ### Playlist generator feedback @@ -187,6 +186,8 @@ tag_fixer.py - Tag fixing engine using plugin metadata update_genres.py - Batch genre tag updater via MusicBrainz validator.py - Verify SoundVault folder layout config.py - Read/write persistent configuration +library_sync_review.py - Library Sync review UI with plan/preview/execute controls +library_sync.py - Library Sync scan/compare logic and plan execution helpers mutagen_stub/ - Minimal fallback used by the tests controllers/ @@ -220,3 +221,8 @@ These items are currently under development and not yet part of the stable relea - Metadata Plugins (Discogs, Spotify) See [`docs/project_documentation.html`](docs/project_documentation.html) for technical details. + +## Known gaps + +- **Tidal-dl sync**: `tidal-dl` is listed in `requirements.txt`, but there is no UI or workflow wired up yet. +- **LLM assistant models**: no GGUF model ships with the repo; you must provide one and update `plugins/assistant_plugin.py` if you want to use the helper. diff --git a/docs/library_sync_redesign.md b/docs/library_sync_redesign.md index 7689c8b..53b19a7 100644 --- a/docs/library_sync_redesign.md +++ b/docs/library_sync_redesign.md @@ -1,92 +1,74 @@ -# Library Sync UI Redesign +# Library Sync UI Redesign (Current Implementation) ## Overview -This redesign reframes the Library Sync tool as a review-first comparison interface between an existing music library and an incoming folder. The UI prioritizes clarity, safety, and trust by clearly distinguishing incoming content from existing content, surfacing duplicates and collisions, and enabling informed decisions through non-destructive review actions. All automated file operations and playlist updates are intentionally excluded from this tool. +The Library Sync UI is a review-first comparison tool for an existing library and an incoming folder. It provides two synchronized lists (Incoming Tracks and Existing Tracks), a text inspector, and an optional plan/preview/execute pipeline for copy/move operations. File operations only occur after the user explicitly runs **Execute**—scans and previews are non-destructive. -**Backend alignment (current state):** The shipping backend in `library_sync.py` already follows this review-first contract. It performs independent scans of the *Existing Library* and *Incoming Folder*, reuses cached fingerprints where size/mtime match, computes new fingerprints in parallel when needed, and matches via indexed shortlists (fingerprint signature ➜ extension ➜ fallback pool). Matches are classified as *New*, *Collision*, *Exact Match*, or *Low Confidence* with a quality label (*Potential Upgrade* / *Keep Existing*) derived from format priority and bitrate. Execution helpers remain opt-in and write dry-run previews plus JSON/HTML audit logs; file operations are blocked unless a plan is explicitly executed. +**Backend alignment (current state):** `library_sync.py` performs independent scans of the *Existing Library* and *Incoming Folder*, reuses cached fingerprints where size/mtime match, computes new fingerprints in parallel when needed, and matches via indexed shortlists (fingerprint signature ➜ extension ➜ fallback pool). Matches are classified as *New*, *Collision*, *Exact Match*, or *Low Confidence* with a quality label (*Potential Upgrade* / *Keep Existing*) derived from format priority and bitrate. `library_sync_review.py` drives the UI, tracks session state, and wires up plan preview and execution. ## User Flow (Start ➜ Finish) -1. **Pick folders:** The user independently selects an *Existing Library* folder and an *Incoming Folder*. -2. **Configure scan:** The user adjusts fingerprint distance thresholds (global or per-format) and optional quality tie-breaker settings, then starts scans for each folder independently. -3. **Scan libraries:** Each folder is scanned separately using indexed matching and cached fingerprints; progress and cancellation are supported. -4. **Review results:** Results are displayed in two primary lists—**Incoming Tracks** and **Existing Tracks**—with status chips indicating matches, collisions, confidence, and quality relationships. -5. **Inspect details:** Selecting a row opens an inspector showing metadata, fingerprint metrics, quality comparison, and the best-matched counterpart when applicable. -6. **Flag actions:** Users flag items for potential copy or replacement as part of a review-only plan; no file operations occur. -7. **Export plan:** The user exports a report summarizing all findings and flags for later manual execution or use by future controlled tools. +1. **Pick folders:** Select an *Existing Library* folder and an *Incoming Folder*. +2. **Configure scan:** Adjust the global threshold, optional per-format overrides, preset name, and report version. +3. **Scan libraries:** Scan each folder independently (both are cancellable). +4. **Review results:** Inspect Incoming/Existing lists and the text inspector for match details. +5. **Build plan (optional):** Choose transfer mode (copy/move), then **Build Plan** and **Preview** to generate the HTML preview. +6. **Execute (optional):** Run **Execute** to apply the plan; optionally emit a playlist of transferred files. +7. **Export logs / save session:** Use **Export Logs…** and **Save Session** to persist review data. ## UI Components and Controls ### Folder Selection -- **Existing Library selector:** Path picker that remembers the last valid library root. -- **Incoming Folder selector:** Independent path picker for new content. +- **Existing Library:** Path picker for the destination library root. +- **Incoming Folder:** Path picker for the new content root. -### Scan Controls -- **Scan Library / Scan Incoming:** Independent scan buttons with cancellable progress. -- **Threshold presets:** Dropdown for common profiles (e.g., strict, normal, loose) mapped to fingerprint distance values. -- **Per-format overrides (optional disclosure):** Table allowing threshold overrides per extension. - -### Primary Lists (exactly two) -- **Incoming Tracks:** Displays all scanned incoming files. Each row is labeled with a status chip such as *New*, *Collision*, *Exact Match*, or *Low Confidence*. -- **Existing Tracks:** Displays all scanned library files. Rows are highlighted when they are the best match for one or more incoming tracks. -- *Incoming Tracks* always contains all incoming files; “new” versus “matched” is indicated via status chips and filters rather than separate lists. - -### Detail Inspector -- Shows detailed information for the selected item, including tags, duration, bitrate (when available), format-based quality score, fingerprint distance, threshold used, and the single best-matched counterpart (lowest distance). If multiple potential matches exist, only the best candidate is actionable; others are informational. -- Optional shortcuts such as “open containing folder” and “play preview” are shown only when supported by the runtime environment. - -**Implementation snapshot:** Each scan produces normalized `TrackRecord` entries (path, ext, bitrate, size, mtime, duration, fingerprint, tags). The inspector can display `MatchResult` payloads that already include the best match, candidate shortlist (with distances), threshold applied, near-miss margin, confidence score, and quality comparison labels. These payloads also flag when a result is partial due to cancellation, allowing the UI to show “Partial scan” states without blocking retries. - -### Status Chips -- Each row may display one or more chips, including *New*, *Collision*, *Exact Match*, *Low Confidence*, *Potential Upgrade*, *Keep Existing*, or *Missing Metadata*. Chips are used consistently throughout the UI to avoid ambiguity. - -### Minimal Action Buttons (non-destructive only) -- **Filter: Collisions Only:** Toggles visibility to only matched items. -- **Sort: Quality Delta:** Sorts incoming tracks by quality difference relative to their matched library counterpart. -- **Flag for Copy:** Marks selected incoming tracks for inclusion in the export plan. -- **Flag for Replace:** Marks the incoming–existing pair (best match only) as a potential replacement candidate. -- **Export Plan / Report:** Generates a CSV or JSON summary of the current review state. -- **Clear Flags:** Removes all review flags without affecting scan results. -- No UI control, context menu, shortcut, or background task performs file moves, replacements, or playlist updates. - -**Execution pipeline (opt-in):** -- **Plan generation:** `compute_library_sync_plan()` builds a deterministic move/route plan by reusing the indexer engine’s routing rules, remapping destinations under the selected library root, and attaching per-file decisions (*COPY*, *REPLACE*, or *SKIP* with reasons). Dry-run previews reuse the indexer HTML renderer for consistency. -- **Execution:** `execute_plan()` consumes the plan, writes JSON audit logs plus executed HTML reports, creates a “LibrarySync_Added_YYYY-mm-dd_hhmm.m3u8” playlist of transferred files (when enabled), and writes backups before replacements. Cancellations are honored between steps; skipped or review-required items are summarized in the report. +### Scan Configuration +- **Global Threshold:** Default fingerprint distance cutoff. +- **Preset Name:** Stored with the scan session for reference. +- **Per-format overrides:** Text box with `ext=threshold` per line. +- **Report Version:** Stored in the scan session metadata. +- **Scan State:** Read-only indicator of the current scan lifecycle state. -### Progress and Status -- Separate progress indicators are shown for library and incoming scans. Each scan can be cancelled independently. A log panel displays scan configuration, counts, skipped files, and warnings. - -## Determining “Incoming” vs “Existing” -### Matching Logic -- Incoming tracks are compared against the existing library using an indexed fingerprint lookup rather than exhaustive pairwise comparison. Cached fingerprints are reused when available. - -### Thresholds -- Per-format thresholds take precedence over the global threshold. Distances below the active threshold are treated as matches; distances above are treated as non-matches. - -### Confidence Indicators -- **Exact Match:** Fingerprint distance equals zero. -- **Collision:** Distance is below the active threshold. -- **Low Confidence:** Distance is above the threshold but within a defined margin (e.g., threshold + fixed delta or percentage), prompting manual review. - -### Quality Comparison -- When a match exists, a quality score is computed using extension priority and bitrate when available. If bitrate is unavailable, extension priority alone is used. Incoming items with higher scores are labeled *Potential Upgrade*; lower-quality matches are labeled *Keep Existing*. - -## Error Handling and Edge Cases -- Scans support large libraries through incremental progress updates, cancellation after the current file completes processing, and reuse of cached fingerprints. -- Missing metadata is tolerated and flagged without blocking classification. -- Unsupported or unreadable files are skipped with logged reasons. -- Cancelled scans preserve partial results and are clearly labeled as *Partial*. -- Path and permission errors surface as inline banners and log entries without disabling retry. -- Threshold inputs are validated; invalid values are rejected with inline feedback and reverted to the last known good configuration. +### Scan Controls +- **Scan / Cancel:** Independent buttons for library and incoming scans. +- **Progress + Phase labels:** Per-scan progress bar, phase text, and partial-result notice on cancellation. -**Performance profiling hooks:** An optional `PerformanceProfile` collector records cache hits, fingerprint computations, shortlist sizes, and scan/match durations. UI surfaces can render these metrics for troubleshooting slow comparisons without affecting the main workflow. +### Plan & Execution +- **Build Plan:** Computes a deterministic move/copy plan. +- **Preview:** Renders the plan preview HTML. +- **Transfer mode toggle:** Switches between **Copy Originals** and **Move Originals**. +- **Execute:** Applies the current plan (requires a matching, previewed plan). +- **Output playlist:** Optionally emit a playlist of transferred files after execution. +- **Open Preview / Cancel:** Open the preview HTML or cancel the active plan task. -## Export Plan Schema (Minimum) -- Exported reports must include incoming path, existing path (nullable), match status, fingerprint distance, threshold used, quality scores, user flags, notes, timestamp, and a scan configuration hash. +### Primary Lists (two panels) +- **Incoming Tracks:** Shows each incoming file with chips and fingerprint distance. +- **Existing Tracks:** Shows each library file and the number of incoming tracks that map to it as best matches. -## Acceptance Criteria -- The UI contains exactly two primary lists: *Incoming Tracks* and *Existing Tracks*. -- No list, button, shortcut, or background process performs copy, replace, or playlist updates. -- Users can independently select folders, configure thresholds, and scan each source. -- Incoming tracks are clearly labeled via chips rather than separated into additional lists. -- Matching, quality comparison, and confidence indicators are visible and consistent. -- Exported reports fully represent the review state and require no file access to generate. -- Scan cancellation preserves completed results and logs without corrupting state. +### Detail Inspector +Read-only text panel summarizing: +- Track metadata and file details. +- Match status, fingerprint distance, threshold used, confidence score, and quality label. + +### Logs +- Scrollable log panel capturing scans, plan actions, and execution events. +- **Export Logs…** writes the log to disk. +- **Save Session** persists the current scan configuration and folders to config. + +## Status Chips (Current) +- **Incoming list:** New, Collision, Exact Match, Low Confidence, Potential Upgrade, Keep Existing, Missing Metadata, Partial. +- **Existing list:** Best Match, Potential Upgrade, Keep Existing, Partial, or Unmatched. + +## Execution Pipeline (Opt-in) +- **Plan generation:** `compute_library_sync_plan()` remaps indexer-derived destinations under the selected library root and attaches per-file decisions (*COPY*, *REPLACE*, or *SKIP* with reasons). +- **Preview:** Renders a dry-run HTML preview via the indexer HTML renderer. +- **Execution:** `execute_library_sync_plan()` writes JSON audit logs plus HTML reports, optionally creates a “LibrarySync_Added_YYYY-mm-dd_hhmm.m3u8” playlist, and writes backups before replacements. Cancellations are honored between steps. + +## Current Gaps (Not Yet Wired) +- **Per-item copy/replace flags** are modeled in `library_sync_review_state.py`, but the UI does not expose buttons or menus to set them yet. +- **Export report UI** is not exposed (the report helpers exist in `library_sync_review_report.py`, but there is no export button). +- **Filters/sorts and quick actions** (e.g., “Collisions Only” or “Sort by Quality Delta”) from earlier design notes are not present. + +## Acceptance Criteria (Current Implementation) +- Two primary lists: *Incoming Tracks* and *Existing Tracks*. +- Independent scan controls with cancellation and partial-result indicators. +- Explicit plan build/preview/execute controls with a transfer-mode toggle. +- Non-destructive review unless **Execute** is run. +- Logs and session persistence are available from the UI. diff --git a/docs/project_documentation.html b/docs/project_documentation.html index e01a6c3..7cb2446 100644 --- a/docs/project_documentation.html +++ b/docs/project_documentation.html @@ -144,15 +144,17 @@

      2.3 Similarity Inspector


      -

      2.5 Library Sync Review Tool (library_sync.py)

      -

      Comparison-first workflow that aligns to the redesign in docs/library_sync_redesign.md:

      +

      2.5 Library Sync Review Tool (library_sync_review.py / library_sync.py)

      +

      Review-first workflow with a two-list UI plus optional plan/preview/execute actions:

      • Runs independent scans for Library and Incoming folders, reusing cached fingerprints when size/mtime match and computing new prints in parallel when needed.
      • Builds normalized TrackRecord entries (path, ext, bitrate, size, mtime, duration, fingerprint, tags) and matches via indexed shortlists: fingerprint signature → extension cohort → fallback pool.
      • Classifies each incoming track as New, Collision, Exact Match, or Low Confidence, and attaches quality labels (Potential Upgrade / Keep Existing) using format priority and bitrate.
      • Exposes full MatchResult payloads with shortlist distances, thresholds used, near-miss margins, and confidence scores so the UI can render detailed inspectors without extra computation.
      • +
      • The UI provides Incoming/Existing lists, a text inspector, log export, and plan controls (Build Plan, Preview, Execute) with a transfer-mode toggle and optional playlist output.
      • Optional profiling (PerformanceProfile) tracks cache hits, fingerprints computed, shortlist sizes, and scan/match timings for troubleshooting.
      • -
      • Opt-in execution path: compute_library_sync_plan() remaps indexer-derived destinations under the chosen library root and emits deterministic plan items (COPY / REPLACE / SKIP with reasons). execute_plan() writes JSON audits + executed HTML reports, creates a playlist of transferred files, and backs up any replaced destination before writing.
      • +
      • Opt-in execution path: compute_library_sync_plan() remaps indexer-derived destinations under the chosen library root and emits deterministic plan items (COPY / REPLACE / SKIP with reasons). build_library_sync_preview() renders the HTML preview, and execute_library_sync_plan() writes JSON audits + executed HTML reports, creates a playlist of transferred files, and backs up any replaced destination before writing.
      • +
      • Current gaps: per-item copy/replace flags and report-export helpers exist in code, but the UI does not expose them yet.

      From 061c93d69b6091da76add58a6fe35c98a116a298 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 4 Jan 2026 15:00:17 -0500 Subject: [PATCH 274/606] Normalize tag values and harden sanitize --- music_indexer_api.py | 11 +++++++++-- utils/audio_metadata_reader.py | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/music_indexer_api.py b/music_indexer_api.py index 4b51cf9..be4ef54 100644 --- a/music_indexer_api.py +++ b/music_indexer_api.py @@ -88,9 +88,16 @@ def progress_callback(_c, _t, _m, _p): return counts # ─── Shared Utility Functions ───────────────────────────────────────── -def sanitize(name: str) -> str: +def sanitize(name: object) -> str: invalid = r'<>:"/\\|?*' - return "".join(c for c in (name or "Unknown") if c not in invalid).strip() or "Unknown" + if name is None: + raw = "Unknown" + else: + try: + raw = str(name) + except Exception: + raw = "Unknown" + return "".join(c for c in raw if c not in invalid).strip() or "Unknown" def collapse_repeats(name: str) -> str: """ diff --git a/utils/audio_metadata_reader.py b/utils/audio_metadata_reader.py index 23c0ed5..942e3b8 100644 --- a/utils/audio_metadata_reader.py +++ b/utils/audio_metadata_reader.py @@ -37,6 +37,37 @@ SIDECAR_ARTWORK_SUFFIXES = (".artwork", ".cover", ".jpg") +def _normalize_text_value(value: object, key: str) -> str | None: + if value is None: + return None + if isinstance(value, bytes): + for encoding in ("utf-8", "utf-16", "latin-1"): + try: + return value.decode(encoding) + except UnicodeDecodeError: + continue + return value.decode("utf-8", errors="replace") + if isinstance(value, (int, float)): + return str(value) + if isinstance(value, str): + cleaned = value.strip() + return cleaned or None + if not isinstance(value, (int, float)): + logger.warning("Unexpected tag type for %s: %s", key, type(value).__name__) + try: + return str(value).strip() or None + except Exception: + logger.warning("Unsupported tag type for %s: %s", key, type(value).__name__) + return None + + +def _normalize_text_tags(tags: Dict[str, object]) -> Dict[str, object]: + normalized = dict(tags) + for key in ("artist", "albumartist", "title", "album", "genre", "year", "date"): + normalized[key] = _normalize_text_value(tags.get(key), key) + return normalized + + def read_sidecar_artwork_bytes(path: str) -> bytes | None: for suffix in SIDECAR_ARTWORK_SUFFIXES: candidate = f"{path}{suffix}" @@ -277,4 +308,4 @@ def read_metadata( def read_tags(path: str) -> Dict[str, object]: """Return just the tag dictionary for ``path``.""" tags, _covers, _error, _reader = read_metadata(path, include_cover=False) - return tags + return _normalize_text_tags(tags) From 5fe5a1f8a37b42dd19ebb3787bf244f1f89bfb24 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 4 Jan 2026 15:22:47 -0500 Subject: [PATCH 275/606] Normalize year metadata for stable routing --- utils/audio_metadata_reader.py | 16 ++-------------- utils/opus_metadata_reader.py | 15 ++------------- utils/year_normalizer.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 27 deletions(-) create mode 100644 utils/year_normalizer.py diff --git a/utils/audio_metadata_reader.py b/utils/audio_metadata_reader.py index 942e3b8..d04dd18 100644 --- a/utils/audio_metadata_reader.py +++ b/utils/audio_metadata_reader.py @@ -8,6 +8,7 @@ from typing import Dict, Iterable, List, Optional, Tuple from utils.path_helpers import ensure_long_path +from utils.year_normalizer import normalize_year logger = logging.getLogger(__name__) @@ -114,18 +115,6 @@ def _parse_int(value: object) -> int | None: return None -def _parse_year(value: object) -> int | None: - if value in (None, ""): - return None - text = str(value) - if len(text) >= 4 and text[:4].isdigit(): - return int(text[:4]) - digits = "".join(ch for ch in text if ch.isdigit()) - if len(digits) >= 4: - return int(digits[:4]) - return None - - def _tags_from_easy(tags: Dict[str, object]) -> Dict[str, object]: values = _blank_tags() values["artist"] = _first_value(tags.get("artist")) @@ -282,8 +271,7 @@ def read_metadata( reader_hint = "easy tags" logger.debug("M4A metadata reader used: %s for %s", reader_hint, path) - if not tags.get("year"): - tags["year"] = _parse_year(tags.get("date")) + tags["year"] = normalize_year(tags.get("year") or tags.get("date")) if not tags.get("track") and tags.get("tracknumber"): tags["track"] = _parse_int(tags.get("tracknumber")) if not tags.get("disc") and tags.get("discnumber"): diff --git a/utils/opus_metadata_reader.py b/utils/opus_metadata_reader.py index 64a91ef..b4fbba6 100644 --- a/utils/opus_metadata_reader.py +++ b/utils/opus_metadata_reader.py @@ -8,6 +8,7 @@ from typing import Dict, List, Optional, Tuple from utils.path_helpers import ensure_long_path +from utils.year_normalizer import normalize_year logger = logging.getLogger(__name__) @@ -64,18 +65,6 @@ def _parse_int(value: object) -> int | None: return None -def _parse_year(value: object) -> int | None: - if value in (None, ""): - return None - text = str(value) - if len(text) >= 4 and text[:4].isdigit(): - return int(text[:4]) - digits = "".join(ch for ch in text if ch.isdigit()) - if len(digits) >= 4: - return int(digits[:4]) - return None - - def _read_sidecar_artwork_bytes(path: str) -> bytes | None: for suffix in SIDECAR_ARTWORK_SUFFIXES: candidate = f"{path}{suffix}" @@ -169,7 +158,7 @@ def read_opus_metadata( tags["track"] = _parse_int(track_raw) if track_raw not in (None, "") else None tags["disc"] = _parse_int(disc_raw) if disc_raw not in (None, "") else None - tags["year"] = _parse_year(tags.get("date")) + tags["year"] = normalize_year(tags.get("date")) cover_payloads = _extract_cover_payloads(raw_tags) if not cover_payloads: diff --git a/utils/year_normalizer.py b/utils/year_normalizer.py new file mode 100644 index 0000000..e9c5878 --- /dev/null +++ b/utils/year_normalizer.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +import re + +UNKNOWN_YEAR = "Unknown" +_YEAR_PATTERN = re.compile(r"(1[0-9]{3}|2[0-9]{3})") + + +def normalize_year(value: object) -> str: + if value is None: + return UNKNOWN_YEAR + if isinstance(value, (int, float)): + return _format_year(int(value)) + try: + text = str(value).strip() + except Exception: + return UNKNOWN_YEAR + if not text: + return UNKNOWN_YEAR + match = _YEAR_PATTERN.search(text) + if not match: + return UNKNOWN_YEAR + return match.group(1) + + +def _format_year(year: int) -> str: + if 1000 <= year <= 2999: + return f"{year:04d}" + return UNKNOWN_YEAR From 96b7a6158b73deabad3f38fd27706d74a8b5fd8f Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 4 Jan 2026 15:25:46 -0500 Subject: [PATCH 276/606] Harden sanitize for metadata inputs --- .../indexer_engine/music_indexer_api.py | 39 +++++++++++++++- music_indexer_api.py | 44 ++++++++++++++---- tests/test_sanitize.py | 45 +++++++++++++++++++ 3 files changed, 118 insertions(+), 10 deletions(-) create mode 100644 tests/test_sanitize.py diff --git a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py index c50d29c..9720076 100644 --- a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py +++ b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py @@ -5,6 +5,7 @@ import shutil # used for relocating special folders import hashlib import base64 +import unicodedata from collections import defaultdict from typing import Dict, List from dry_run_coordinator import DryRunCoordinator @@ -93,9 +94,43 @@ def progress_callback(_c, _t, _m, _p): return counts # ─── Shared Utility Functions ───────────────────────────────────────── -def sanitize(name: str) -> str: +def _coerce_to_string(value: object) -> str: + if value is None: + return "" + if isinstance(value, bytes): + return value.decode("utf-8", errors="replace") + if isinstance(value, (list, tuple, set)): + parts = [] + for item in value: + try: + part = "" if item is None else str(item) + except Exception: + part = "" + if part: + parts.append(part) + return ", ".join(parts) + try: + return str(value) + except Exception: + return "" + + +def sanitize(name: object) -> str: + """Normalize values for filesystem path segments and HTML/report labels. + + Contract: + - Accepts any object (including None, numbers, lists). + - Coerces to string deterministically. + - Removes filesystem-invalid characters and control chars. + - Returns "Unknown" when the result is empty. + - Does not HTML-escape to avoid double-escaping upstream content. + """ invalid = r'<>:"/\\|?*' - return "".join(c for c in (name or "Unknown") if c not in invalid).strip() or "Unknown" + raw = _coerce_to_string(name) or "Unknown" + normalized = unicodedata.normalize("NFKC", raw) + normalized = re.sub(r"[\x00-\x1f\x7f]", "", normalized) + cleaned = "".join(c for c in normalized if c not in invalid).strip() + return cleaned or "Unknown" def collapse_repeats(name: str) -> str: """ diff --git a/music_indexer_api.py b/music_indexer_api.py index be4ef54..a6b654d 100644 --- a/music_indexer_api.py +++ b/music_indexer_api.py @@ -5,6 +5,7 @@ import shutil # used for relocating special folders import hashlib import base64 +import unicodedata from collections import defaultdict from typing import Dict, List from dry_run_coordinator import DryRunCoordinator @@ -88,16 +89,43 @@ def progress_callback(_c, _t, _m, _p): return counts # ─── Shared Utility Functions ───────────────────────────────────────── +def _coerce_to_string(value: object) -> str: + if value is None: + return "" + if isinstance(value, bytes): + return value.decode("utf-8", errors="replace") + if isinstance(value, (list, tuple, set)): + parts = [] + for item in value: + try: + part = "" if item is None else str(item) + except Exception: + part = "" + if part: + parts.append(part) + return ", ".join(parts) + try: + return str(value) + except Exception: + return "" + + def sanitize(name: object) -> str: + """Normalize values for filesystem path segments and HTML/report labels. + + Contract: + - Accepts any object (including None, numbers, lists). + - Coerces to string deterministically. + - Removes filesystem-invalid characters and control chars. + - Returns "Unknown" when the result is empty. + - Does not HTML-escape to avoid double-escaping upstream content. + """ invalid = r'<>:"/\\|?*' - if name is None: - raw = "Unknown" - else: - try: - raw = str(name) - except Exception: - raw = "Unknown" - return "".join(c for c in raw if c not in invalid).strip() or "Unknown" + raw = _coerce_to_string(name) or "Unknown" + normalized = unicodedata.normalize("NFKC", raw) + normalized = re.sub(r"[\x00-\x1f\x7f]", "", normalized) + cleaned = "".join(c for c in normalized if c not in invalid).strip() + return cleaned or "Unknown" def collapse_repeats(name: str) -> str: """ diff --git a/tests/test_sanitize.py b/tests/test_sanitize.py new file mode 100644 index 0000000..143baa9 --- /dev/null +++ b/tests/test_sanitize.py @@ -0,0 +1,45 @@ +from music_indexer_api import sanitize +import importlib.util +from pathlib import Path + + +def _load_engine_sanitize(): + module_path = ( + Path(__file__).resolve().parents[1] + / "library_sync_indexer_engine" + / "indexer_engine" + / "music_indexer_api.py" + ) + spec = importlib.util.spec_from_file_location( + "library_sync_indexer_engine.indexer_engine.music_indexer_api", + module_path, + ) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module.sanitize + + +def test_sanitize_handles_non_string_inputs(): + assert sanitize(None) == "Unknown" + assert sanitize(2024) == "2024" + assert sanitize(3.14) == "3.14" + assert sanitize(["AC/DC", None, "Live"]) == "ACDC, Live" + + +def test_sanitize_preserves_entities_without_double_escaping(): + assert sanitize("A & B") == "A & B" + assert sanitize("Rock & Roll") == "Rock & Roll" + + +def test_sanitize_strips_invalid_characters_and_whitespace(): + assert sanitize(" <> ") == "Unknown" + assert sanitize("A/B") == "ABC" + + +def test_engine_sanitize_matches_api_sanitize(): + engine_sanitize = _load_engine_sanitize() + samples = [None, 2024, 3.14, ["AC/DC", None, "Live"], "A/B", "Rock & Roll"] + assert [engine_sanitize(sample) for sample in samples] == [ + sanitize(sample) for sample in samples + ] From 174e496022d3c0b48c7fb2e3e0451e20abe3ee4e Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 4 Jan 2026 15:29:59 -0500 Subject: [PATCH 277/606] Harden metadata normalization in indexer pipeline --- .../indexer_engine/music_indexer_api.py | 138 +++++++++++++----- music_indexer_api.py | 138 +++++++++++++----- 2 files changed, 204 insertions(+), 72 deletions(-) diff --git a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py index 9720076..4e759ce 100644 --- a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py +++ b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py @@ -43,7 +43,7 @@ class ID3NoHeaderError(Exception): FINGERPRINT_TIMEOUT = 30 # seconds before fingerprint worker is timed out # ─── Helper: build primary_counts for the entire vault ───────────────────── -def build_primary_counts(root_path, progress_callback=None, phase="A"): +def build_primary_counts(root_path, progress_callback=None, phase="A", log_callback=None): """ Walk the entire vault (By Artist, By Year, Incoming, etc.) and return a dict mapping each lowercase-normalized artist → total file count. @@ -56,6 +56,9 @@ def build_primary_counts(root_path, progress_callback=None, phase="A"): if progress_callback is None: def progress_callback(_c, _t, _m, _p): pass + if log_callback is None: + def log_callback(_m): + pass counts = {} idx = 0 @@ -76,7 +79,7 @@ def progress_callback(_c, _t, _m, _p): progress_callback(idx, 0, os.path.relpath(path, root_path), phase) # Normalize artist identically to Phase 4 - raw = (tags.get("artist") or "").strip() + raw = _normalize_tag_value(tags.get("artist"), "artist", log_callback) if not raw: name_only = os.path.splitext(fname)[0] @@ -132,6 +135,40 @@ def sanitize(name: object) -> str: cleaned = "".join(c for c in normalized if c not in invalid).strip() return cleaned or "Unknown" +def _normalize_tag_value(value: object, field: str, log_callback=None) -> str: + text = _coerce_to_string(value).strip() + if not text: + if log_callback and value not in (None, "", [], (), set()): + log_callback(f" ! {field} tag value {value!r} invalid; treating as Unknown") + return "" + if log_callback and not isinstance(value, str): + log_callback(f" ! {field} tag value {value!r} coerced to '{text}'") + return text + + +def _sanitize_tag_value(value: object, field: str, log_callback=None) -> str: + normalized = _normalize_tag_value(value, field, log_callback) + sanitized = sanitize(normalized) if normalized else "Unknown" + if log_callback and normalized and sanitized == "Unknown": + log_callback(f" ! {field} tag '{normalized}' sanitized to 'Unknown'") + return sanitized + + +def _ensure_unique_destination(path: str, moves: dict, log_callback=None) -> str: + if path not in moves.values() and not os.path.exists(path): + return path + base, ext = os.path.splitext(path) + counter = 1 + candidate = f"{base} ({counter}){ext}" + while candidate in moves.values() or os.path.exists(candidate): + counter += 1 + candidate = f"{base} ({counter}){ext}" + if log_callback: + log_callback( + f" ! Destination collision detected; renamed to '{os.path.basename(candidate)}'" + ) + return candidate + def collapse_repeats(name: str) -> str: """ If name is something like 'DROELOEDROELOE', collapse to 'DROELOE'. @@ -222,6 +259,7 @@ def extract_primary_and_collabs(raw_artist: str): - Else split on uppercase boundary, treat first as primary and rest as a single collab string. - Otherwise, primary = raw_artist, collabs = []. """ + raw_artist = _coerce_to_string(raw_artist) if not raw_artist: return ("Unknown", []) text = raw_artist.strip() @@ -386,7 +424,7 @@ def progress_callback(current, total, message, phase): _flush(db_path) # --- Phase 0: Pre-scan entire vault (under MUSIC_ROOT) --- - global_counts = build_primary_counts(MUSIC_ROOT, progress_callback, phase="A") + global_counts = build_primary_counts(MUSIC_ROOT, progress_callback, phase="A", log_callback=log_callback) log_callback(f" → Pre-scan: found {len(global_counts)} unique artists") log_callback(f" → DEBUG: MUSIC_ROOT = {MUSIC_ROOT}") log_callback(f" → DEBUG: droeloe count = {global_counts.get('droeloe', 0)}") @@ -438,13 +476,19 @@ def _compute(path: str) -> tuple[int | None, str | None]: if idx % 50 == 0 or idx == total_audio: log_callback(f" • Processing file {idx}/{total_audio} for dedupe") data = get_tags(fullpath) - raw_artist = data["artist"] or os.path.splitext(os.path.basename(fullpath))[0] + raw_artist = _normalize_tag_value(data.get("artist"), "artist", log_callback) + if not raw_artist: + raw_artist = os.path.splitext(os.path.basename(fullpath))[0] raw_artist = collapse_repeats(raw_artist) - title = data["title"] or os.path.splitext(os.path.basename(fullpath))[0] - album = data["album"] or "" + title = _normalize_tag_value(data.get("title"), "title", log_callback) + if not title: + title = os.path.splitext(os.path.basename(fullpath))[0] + album = _normalize_tag_value(data.get("album"), "album", log_callback) + year = _normalize_tag_value(data.get("year"), "year", log_callback) + genre = _normalize_tag_value(data.get("genre"), "genre", log_callback) meta_count = sum( 1 - for t in [data.get("artist"), data.get("title"), data.get("album"), data.get("track"), data.get("year"), data.get("genre")] + for t in [raw_artist, title, album, data.get("track"), year, genre] if t ) primary, _ = extract_primary_and_collabs(raw_artist) @@ -576,14 +620,18 @@ def _compute(path: str) -> tuple[int | None, str | None]: progress_callback(idx, total_kept, f"Metadata {os.path.relpath(fullpath, root_path)}", "B") data = get_tags(fullpath) - raw_artist = data["artist"] or os.path.splitext(os.path.basename(fullpath))[0] + raw_artist = _normalize_tag_value(data.get("artist"), "artist", log_callback) + if not raw_artist: + raw_artist = os.path.splitext(os.path.basename(fullpath))[0] raw_artist = collapse_repeats(raw_artist) - title = data["title"] or os.path.splitext(os.path.basename(fullpath))[0] - album = data["album"] - year = data["year"] - genre = data["genre"] - track = data["track"] - part_of_set = data["part_of_set"] + title = _normalize_tag_value(data.get("title"), "title", log_callback) + if not title: + title = os.path.splitext(os.path.basename(fullpath))[0] + album = _normalize_tag_value(data.get("album"), "album", log_callback) + year = _normalize_tag_value(data.get("year"), "year", log_callback) + genre = _normalize_tag_value(data.get("genre"), "genre", log_callback) + track = data.get("track") + part_of_set = _normalize_tag_value(data.get("part_of_set"), "part_of_set", log_callback) # 1) Primary & collabs (preserve original case) primary, collabs = extract_primary_and_collabs(raw_artist) @@ -634,7 +682,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: "part_of_set": part_of_set, "cover_hash": cover_hash, "folder_tags": folder_tags, - "missing_core": not data.get("artist") or not data.get("title"), + "missing_core": not raw_artist or not title, } log_callback(f" → Collected metadata for {total_kept} files.") @@ -656,12 +704,13 @@ def _compute(path: str) -> tuple[int | None, str | None]: log_callback(f" • Determining destination {index}/{total}") raw_artist = info["raw_artist"] - title = info["title"] or "" - album = info["album"] or "" - year = info["year"] or "Unknown" - track = info["track"] + title = info["title"] or "" + album = info["album"] or "" + year = info["year"] or "" + track = info["track"] part_of_set = info.get("part_of_set") folders = info["folder_tags"] + year_label = _sanitize_tag_value(year, "year", log_callback) if info.get("missing_core"): candidate = os.path.join(review_root, os.path.basename(old_path)) root_c, ext_c = os.path.splitext(candidate) @@ -681,13 +730,15 @@ def _compute(path: str) -> tuple[int | None, str | None]: # Lookup how many total tracks this artist has across the entire vault count_now = global_counts.get(p_lower, 0) - decision_log.append(f"Song: {raw_artist} – {title} ({album}) → global_counts[{raw_artist}] = {count_now}") + decision_log.append( + f"Song: {raw_artist} – {title} ({album}) → global_counts[{raw_artist}] = {count_now}" + ) # ─── 4.1) EARLY BAIL-OUT: “Rare” artists go straight to By Year ───────────────── if count_now < COMMON_ARTIST_THRESHOLD: - base_folder = os.path.join(MUSIC_ROOT, "By Year", sanitize(year)) + base_folder = os.path.join(MUSIC_ROOT, "By Year", year_label) decision_log.append( - f" Early-exit: Count {count_now} < {COMMON_ARTIST_THRESHOLD} → group under By Year/{year}" + f" Early-exit: Count {count_now} < {COMMON_ARTIST_THRESHOLD} → group under By Year/{year_label}" ) # Build the new filename as usual (rename invalid artists or use sanitized tags) @@ -696,13 +747,14 @@ def _compute(path: str) -> tuple[int | None, str | None]: decision_log.append(f" Raw artist '{raw_artist}' malformed → keeping filename '{new_filename}'") else: ext = os.path.splitext(old_path)[1].lower() - filename_artist = sanitize(raw_artist) + filename_artist = _sanitize_tag_value(raw_artist, "artist", log_callback) track_str = f"{track:02d}" if track is not None else "00" - title_str = sanitize(title) + title_str = _sanitize_tag_value(title, "title", log_callback) new_filename = f"{filename_artist}_{track_str}_{title_str}{ext}" decision_log.append(f" Renaming to '{new_filename}' (using raw artist)") new_path = os.path.join(base_folder, new_filename) + new_path = _ensure_unique_destination(new_path, moves, log_callback) moves[old_path] = new_path decision_log.append( f" → Final: '{os.path.relpath(new_path, MUSIC_ROOT)}'" @@ -727,8 +779,10 @@ def _compute(path: str) -> tuple[int | None, str | None]: f" → Enough remixes ({rcount} ≥ {REMIX_FOLDER_THRESHOLD}) " f"AND count_now ({count_now}) ≥ {COMMON_ARTIST_THRESHOLD}; force primary = '{main_artist}'" ) - artist_folder = os.path.join(MUSIC_ROOT, "By Artist", sanitize(main_artist)) - base_folder = os.path.join(artist_folder, sanitize(album)) + artist_folder = os.path.join( + MUSIC_ROOT, "By Artist", _sanitize_tag_value(main_artist, "artist", log_callback) + ) + base_folder = os.path.join(artist_folder, _sanitize_tag_value(album, "album", log_callback)) base_folder = _apply_part_of_set(base_folder, album, part_of_set, decision_log) # Build filename (skip normal album logic) @@ -738,13 +792,14 @@ def _compute(path: str) -> tuple[int | None, str | None]: decision_log.append(f" Raw artist '{raw_artist}' malformed → keeping '{basename}'") else: ext = os.path.splitext(old_path)[1].lower() - filename_artist = sanitize(raw_artist) + filename_artist = _sanitize_tag_value(raw_artist, "artist", log_callback) track_str = f"{track:02d}" if track is not None else "00" - title_str = sanitize(title) + title_str = _sanitize_tag_value(title, "title", log_callback) new_filename = f"{filename_artist}_{track_str}_{title_str}{ext}" decision_log.append(f" Renaming to '{new_filename}' (using raw artist)") new_path = os.path.join(base_folder, new_filename) + new_path = _ensure_unique_destination(new_path, moves, log_callback) moves[old_path] = new_path decision_log.append( f" → (Remixes folder) placed under By Artist/{main_artist}/{album} " @@ -768,13 +823,16 @@ def _compute(path: str) -> tuple[int | None, str | None]: decision_log.append(f" → After ranking, primary = '{primary}'") # Build “By Artist/” base path - artist_folder = os.path.join(MUSIC_ROOT, "By Artist", sanitize(primary)) + artist_folder = os.path.join( + MUSIC_ROOT, "By Artist", _sanitize_tag_value(primary, "artist", log_callback) + ) # ─── 4.4) COMMON ARTIST: decide between “Album” vs. “Singles” ─────────────── # (At this point, count_now ≥ COMMON_ARTIST_THRESHOLD is guaranteed.) if not album or album.strip().lower() == title.strip().lower(): # No album tag (or album == title) → put into “ - Singles” - base_folder = os.path.join(artist_folder, f"{sanitize(primary)} - Singles") + primary_label = _sanitize_tag_value(primary, "artist", log_callback) + base_folder = os.path.join(artist_folder, f"{primary_label} - Singles") decision_log.append( f" Count {count_now} ≥ {COMMON_ARTIST_THRESHOLD}, no album or album=title " f"→ group under '{primary} - Singles'" @@ -783,14 +841,17 @@ def _compute(path: str) -> tuple[int | None, str | None]: c2 = album_counts.get((p_lower, album.lower()), 0) if c2 > 3: # Enough tracks in this album → put into “/” - base_folder = os.path.join(artist_folder, sanitize(album)) + base_folder = os.path.join( + artist_folder, _sanitize_tag_value(album, "album", log_callback) + ) decision_log.append( f" Count {count_now} ≥ {COMMON_ARTIST_THRESHOLD}, album_count={c2} > 3 " f"→ group under Album '{album}'" ) else: # Few tracks in album → treat as a “Singles” release - base_folder = os.path.join(artist_folder, f"{sanitize(primary)} - Singles") + primary_label = _sanitize_tag_value(primary, "artist", log_callback) + base_folder = os.path.join(artist_folder, f"{primary_label} - Singles") decision_log.append( f" Count {count_now} ≥ {COMMON_ARTIST_THRESHOLD}, but " f"album_count={c2} ≤ 3 → group under '{primary} - Singles'" @@ -805,13 +866,14 @@ def _compute(path: str) -> tuple[int | None, str | None]: decision_log.append(f" Raw artist '{raw_artist}' malformed → keeping '{basename}'") else: ext = os.path.splitext(old_path)[1].lower() - filename_artist = sanitize(raw_artist) + filename_artist = _sanitize_tag_value(raw_artist, "artist", log_callback) track_str = f"{track:02d}" if track is not None else "00" - title_str = sanitize(title) + title_str = _sanitize_tag_value(title, "title", log_callback) new_filename = f"{filename_artist}_{track_str}_{title_str}{ext}" decision_log.append(f" Renaming to '{new_filename}' (using raw artist)") new_path = os.path.join(base_folder, new_filename) + new_path = _ensure_unique_destination(new_path, moves, log_callback) moves[old_path] = new_path decision_log.append( f" → Final: '{os.path.relpath(new_path, MUSIC_ROOT)}'" @@ -824,12 +886,16 @@ def _compute(path: str) -> tuple[int | None, str | None]: leftover.discard(album) if part_of_set: leftover.discard(part_of_set) - leftover.discard(year) + if year: + leftover.discard(year) if genre: leftover.add(genre) + sanitized_leftover = sorted( + {_sanitize_tag_value(tag, "tag", log_callback) for tag in leftover} + ) tag_index[new_path] = { - "leftover_tags": sorted(leftover), + "leftover_tags": sanitized_leftover, "old_paths": sorted(folders) } diff --git a/music_indexer_api.py b/music_indexer_api.py index a6b654d..3aef255 100644 --- a/music_indexer_api.py +++ b/music_indexer_api.py @@ -38,7 +38,7 @@ def MutagenFile(*_a, **_k): FINGERPRINT_TIMEOUT = 30 # seconds before fingerprint worker is timed out # ─── Helper: build primary_counts for the entire vault ───────────────────── -def build_primary_counts(root_path, progress_callback=None, phase="A"): +def build_primary_counts(root_path, progress_callback=None, phase="A", log_callback=None): """ Walk the entire vault (By Artist, By Year, Incoming, etc.) and return a dict mapping each lowercase-normalized artist → total file count. @@ -51,6 +51,9 @@ def build_primary_counts(root_path, progress_callback=None, phase="A"): if progress_callback is None: def progress_callback(_c, _t, _m, _p): pass + if log_callback is None: + def log_callback(_m): + pass counts = {} idx = 0 @@ -71,7 +74,7 @@ def progress_callback(_c, _t, _m, _p): progress_callback(idx, 0, os.path.relpath(path, root_path), phase) # Normalize artist identically to Phase 4 - raw = (tags.get("artist") or "").strip() + raw = _normalize_tag_value(tags.get("artist"), "artist", log_callback) if not raw: name_only = os.path.splitext(fname)[0] @@ -127,6 +130,40 @@ def sanitize(name: object) -> str: cleaned = "".join(c for c in normalized if c not in invalid).strip() return cleaned or "Unknown" +def _normalize_tag_value(value: object, field: str, log_callback=None) -> str: + text = _coerce_to_string(value).strip() + if not text: + if log_callback and value not in (None, "", [], (), set()): + log_callback(f" ! {field} tag value {value!r} invalid; treating as Unknown") + return "" + if log_callback and not isinstance(value, str): + log_callback(f" ! {field} tag value {value!r} coerced to '{text}'") + return text + + +def _sanitize_tag_value(value: object, field: str, log_callback=None) -> str: + normalized = _normalize_tag_value(value, field, log_callback) + sanitized = sanitize(normalized) if normalized else "Unknown" + if log_callback and normalized and sanitized == "Unknown": + log_callback(f" ! {field} tag '{normalized}' sanitized to 'Unknown'") + return sanitized + + +def _ensure_unique_destination(path: str, moves: dict, log_callback=None) -> str: + if path not in moves.values() and not os.path.exists(path): + return path + base, ext = os.path.splitext(path) + counter = 1 + candidate = f"{base} ({counter}){ext}" + while candidate in moves.values() or os.path.exists(candidate): + counter += 1 + candidate = f"{base} ({counter}){ext}" + if log_callback: + log_callback( + f" ! Destination collision detected; renamed to '{os.path.basename(candidate)}'" + ) + return candidate + def collapse_repeats(name: str) -> str: """ If name is something like 'DROELOEDROELOE', collapse to 'DROELOE'. @@ -217,6 +254,7 @@ def extract_primary_and_collabs(raw_artist: str): - Else split on uppercase boundary, treat first as primary and rest as a single collab string. - Otherwise, primary = raw_artist, collabs = []. """ + raw_artist = _coerce_to_string(raw_artist) if not raw_artist: return ("Unknown", []) text = raw_artist.strip() @@ -381,7 +419,7 @@ def progress_callback(current, total, message, phase): _flush(db_path) # --- Phase 0: Pre-scan entire vault (under MUSIC_ROOT) --- - global_counts = build_primary_counts(MUSIC_ROOT, progress_callback, phase="A") + global_counts = build_primary_counts(MUSIC_ROOT, progress_callback, phase="A", log_callback=log_callback) log_callback(f" → Pre-scan: found {len(global_counts)} unique artists") log_callback(f" → DEBUG: MUSIC_ROOT = {MUSIC_ROOT}") log_callback(f" → DEBUG: droeloe count = {global_counts.get('droeloe', 0)}") @@ -433,13 +471,19 @@ def _compute(path: str) -> tuple[int | None, str | None]: if idx % 50 == 0 or idx == total_audio: log_callback(f" • Processing file {idx}/{total_audio} for dedupe") data = get_tags(fullpath) - raw_artist = data["artist"] or os.path.splitext(os.path.basename(fullpath))[0] + raw_artist = _normalize_tag_value(data.get("artist"), "artist", log_callback) + if not raw_artist: + raw_artist = os.path.splitext(os.path.basename(fullpath))[0] raw_artist = collapse_repeats(raw_artist) - title = data["title"] or os.path.splitext(os.path.basename(fullpath))[0] - album = data["album"] or "" + title = _normalize_tag_value(data.get("title"), "title", log_callback) + if not title: + title = os.path.splitext(os.path.basename(fullpath))[0] + album = _normalize_tag_value(data.get("album"), "album", log_callback) + year = _normalize_tag_value(data.get("year"), "year", log_callback) + genre = _normalize_tag_value(data.get("genre"), "genre", log_callback) meta_count = sum( 1 - for t in [data.get("artist"), data.get("title"), data.get("album"), data.get("track"), data.get("year"), data.get("genre")] + for t in [raw_artist, title, album, data.get("track"), year, genre] if t ) primary, _ = extract_primary_and_collabs(raw_artist) @@ -571,14 +615,18 @@ def _compute(path: str) -> tuple[int | None, str | None]: progress_callback(idx, total_kept, f"Metadata {os.path.relpath(fullpath, root_path)}", "B") data = get_tags(fullpath) - raw_artist = data["artist"] or os.path.splitext(os.path.basename(fullpath))[0] + raw_artist = _normalize_tag_value(data.get("artist"), "artist", log_callback) + if not raw_artist: + raw_artist = os.path.splitext(os.path.basename(fullpath))[0] raw_artist = collapse_repeats(raw_artist) - title = data["title"] or os.path.splitext(os.path.basename(fullpath))[0] - album = data["album"] - year = data["year"] - genre = data["genre"] - track = data["track"] - part_of_set = data["part_of_set"] + title = _normalize_tag_value(data.get("title"), "title", log_callback) + if not title: + title = os.path.splitext(os.path.basename(fullpath))[0] + album = _normalize_tag_value(data.get("album"), "album", log_callback) + year = _normalize_tag_value(data.get("year"), "year", log_callback) + genre = _normalize_tag_value(data.get("genre"), "genre", log_callback) + track = data.get("track") + part_of_set = _normalize_tag_value(data.get("part_of_set"), "part_of_set", log_callback) # 1) Primary & collabs (preserve original case) primary, collabs = extract_primary_and_collabs(raw_artist) @@ -618,7 +666,7 @@ def _compute(path: str) -> tuple[int | None, str | None]: "part_of_set": part_of_set, "cover_hash": cover_hash, "folder_tags": folder_tags, - "missing_core": not data.get("artist") or not data.get("title"), + "missing_core": not raw_artist or not title, } log_callback(f" → Collected metadata for {total_kept} files.") @@ -640,12 +688,13 @@ def _compute(path: str) -> tuple[int | None, str | None]: log_callback(f" • Determining destination {index}/{total}") raw_artist = info["raw_artist"] - title = info["title"] or "" - album = info["album"] or "" - year = info["year"] or "Unknown" - track = info["track"] + title = info["title"] or "" + album = info["album"] or "" + year = info["year"] or "" + track = info["track"] part_of_set = info.get("part_of_set") folders = info["folder_tags"] + year_label = _sanitize_tag_value(year, "year", log_callback) if info.get("missing_core"): candidate = os.path.join(review_root, os.path.basename(old_path)) root_c, ext_c = os.path.splitext(candidate) @@ -665,13 +714,15 @@ def _compute(path: str) -> tuple[int | None, str | None]: # Lookup how many total tracks this artist has across the entire vault count_now = global_counts.get(p_lower, 0) - decision_log.append(f"Song: {raw_artist} – {title} ({album}) → global_counts[{raw_artist}] = {count_now}") + decision_log.append( + f"Song: {raw_artist} – {title} ({album}) → global_counts[{raw_artist}] = {count_now}" + ) # ─── 4.1) EARLY BAIL-OUT: “Rare” artists go straight to By Year ───────────────── if count_now < COMMON_ARTIST_THRESHOLD: - base_folder = os.path.join(MUSIC_ROOT, "By Year", sanitize(year)) + base_folder = os.path.join(MUSIC_ROOT, "By Year", year_label) decision_log.append( - f" Early-exit: Count {count_now} < {COMMON_ARTIST_THRESHOLD} → group under By Year/{year}" + f" Early-exit: Count {count_now} < {COMMON_ARTIST_THRESHOLD} → group under By Year/{year_label}" ) # Build the new filename as usual (rename invalid artists or use sanitized tags) @@ -680,13 +731,14 @@ def _compute(path: str) -> tuple[int | None, str | None]: decision_log.append(f" Raw artist '{raw_artist}' malformed → keeping filename '{new_filename}'") else: ext = os.path.splitext(old_path)[1].lower() - filename_artist = sanitize(raw_artist) + filename_artist = _sanitize_tag_value(raw_artist, "artist", log_callback) track_str = f"{track:02d}" if track is not None else "00" - title_str = sanitize(title) + title_str = _sanitize_tag_value(title, "title", log_callback) new_filename = f"{filename_artist}_{track_str}_{title_str}{ext}" decision_log.append(f" Renaming to '{new_filename}' (using raw artist)") new_path = os.path.join(base_folder, new_filename) + new_path = _ensure_unique_destination(new_path, moves, log_callback) moves[old_path] = new_path decision_log.append( f" → Final: '{os.path.relpath(new_path, MUSIC_ROOT)}'" @@ -711,8 +763,10 @@ def _compute(path: str) -> tuple[int | None, str | None]: f" → Enough remixes ({rcount} ≥ {REMIX_FOLDER_THRESHOLD}) " f"AND count_now ({count_now}) ≥ {COMMON_ARTIST_THRESHOLD}; force primary = '{main_artist}'" ) - artist_folder = os.path.join(MUSIC_ROOT, "By Artist", sanitize(main_artist)) - base_folder = os.path.join(artist_folder, sanitize(album)) + artist_folder = os.path.join( + MUSIC_ROOT, "By Artist", _sanitize_tag_value(main_artist, "artist", log_callback) + ) + base_folder = os.path.join(artist_folder, _sanitize_tag_value(album, "album", log_callback)) base_folder = _apply_part_of_set(base_folder, album, part_of_set, decision_log) # Build filename (skip normal album logic) @@ -722,13 +776,14 @@ def _compute(path: str) -> tuple[int | None, str | None]: decision_log.append(f" Raw artist '{raw_artist}' malformed → keeping '{basename}'") else: ext = os.path.splitext(old_path)[1].lower() - filename_artist = sanitize(raw_artist) + filename_artist = _sanitize_tag_value(raw_artist, "artist", log_callback) track_str = f"{track:02d}" if track is not None else "00" - title_str = sanitize(title) + title_str = _sanitize_tag_value(title, "title", log_callback) new_filename = f"{filename_artist}_{track_str}_{title_str}{ext}" decision_log.append(f" Renaming to '{new_filename}' (using raw artist)") new_path = os.path.join(base_folder, new_filename) + new_path = _ensure_unique_destination(new_path, moves, log_callback) moves[old_path] = new_path decision_log.append( f" → (Remixes folder) placed under By Artist/{main_artist}/{album} " @@ -752,13 +807,16 @@ def _compute(path: str) -> tuple[int | None, str | None]: decision_log.append(f" → After ranking, primary = '{primary}'") # Build “By Artist/” base path - artist_folder = os.path.join(MUSIC_ROOT, "By Artist", sanitize(primary)) + artist_folder = os.path.join( + MUSIC_ROOT, "By Artist", _sanitize_tag_value(primary, "artist", log_callback) + ) # ─── 4.4) COMMON ARTIST: decide between “Album” vs. “Singles” ─────────────── # (At this point, count_now ≥ COMMON_ARTIST_THRESHOLD is guaranteed.) if not album or album.strip().lower() == title.strip().lower(): # No album tag (or album == title) → put into “ - Singles” - base_folder = os.path.join(artist_folder, f"{sanitize(primary)} - Singles") + primary_label = _sanitize_tag_value(primary, "artist", log_callback) + base_folder = os.path.join(artist_folder, f"{primary_label} - Singles") decision_log.append( f" Count {count_now} ≥ {COMMON_ARTIST_THRESHOLD}, no album or album=title " f"→ group under '{primary} - Singles'" @@ -767,14 +825,17 @@ def _compute(path: str) -> tuple[int | None, str | None]: c2 = album_counts.get((p_lower, album.lower()), 0) if c2 > 3: # Enough tracks in this album → put into “/” - base_folder = os.path.join(artist_folder, sanitize(album)) + base_folder = os.path.join( + artist_folder, _sanitize_tag_value(album, "album", log_callback) + ) decision_log.append( f" Count {count_now} ≥ {COMMON_ARTIST_THRESHOLD}, album_count={c2} > 3 " f"→ group under Album '{album}'" ) else: # Few tracks in album → treat as a “Singles” release - base_folder = os.path.join(artist_folder, f"{sanitize(primary)} - Singles") + primary_label = _sanitize_tag_value(primary, "artist", log_callback) + base_folder = os.path.join(artist_folder, f"{primary_label} - Singles") decision_log.append( f" Count {count_now} ≥ {COMMON_ARTIST_THRESHOLD}, but " f"album_count={c2} ≤ 3 → group under '{primary} - Singles'" @@ -789,13 +850,14 @@ def _compute(path: str) -> tuple[int | None, str | None]: decision_log.append(f" Raw artist '{raw_artist}' malformed → keeping '{basename}'") else: ext = os.path.splitext(old_path)[1].lower() - filename_artist = sanitize(raw_artist) + filename_artist = _sanitize_tag_value(raw_artist, "artist", log_callback) track_str = f"{track:02d}" if track is not None else "00" - title_str = sanitize(title) + title_str = _sanitize_tag_value(title, "title", log_callback) new_filename = f"{filename_artist}_{track_str}_{title_str}{ext}" decision_log.append(f" Renaming to '{new_filename}' (using raw artist)") new_path = os.path.join(base_folder, new_filename) + new_path = _ensure_unique_destination(new_path, moves, log_callback) moves[old_path] = new_path decision_log.append( f" → Final: '{os.path.relpath(new_path, MUSIC_ROOT)}'" @@ -808,12 +870,16 @@ def _compute(path: str) -> tuple[int | None, str | None]: leftover.discard(album) if part_of_set: leftover.discard(part_of_set) - leftover.discard(year) + if year: + leftover.discard(year) if genre: leftover.add(genre) + sanitized_leftover = sorted( + {_sanitize_tag_value(tag, "tag", log_callback) for tag in leftover} + ) tag_index[new_path] = { - "leftover_tags": sorted(leftover), + "leftover_tags": sanitized_leftover, "old_paths": sorted(folders) } From 9da4e49338440d4a873a8aca8e7af300a255284b Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 4 Jan 2026 15:35:13 -0500 Subject: [PATCH 278/606] Refocus indexer on routing without dedupe/artwork --- music_indexer_api.py | 387 ++++++++----------------------------------- 1 file changed, 68 insertions(+), 319 deletions(-) diff --git a/music_indexer_api.py b/music_indexer_api.py index 3aef255..c2a3b1a 100644 --- a/music_indexer_api.py +++ b/music_indexer_api.py @@ -3,13 +3,10 @@ import os import re import shutil # used for relocating special folders -import hashlib -import base64 import unicodedata from collections import defaultdict -from typing import Dict, List +from typing import List from dry_run_coordinator import DryRunCoordinator -from config import MIXED_CODEC_THRESHOLD_BOOST, load_config def _verify_dependencies() -> None: """Raise RuntimeError if the real mutagen library is missing.""" @@ -26,16 +23,13 @@ def _verify_dependencies() -> None: except Exception: # pragma: no cover - optional dependency def MutagenFile(*_a, **_k): return None -from utils.audio_metadata_reader import read_metadata, read_tags +from utils.audio_metadata_reader import read_tags from indexer_control import check_cancelled, cancel_event # ─── CONFIGURATION ───────────────────────────────────────────────────── COMMON_ARTIST_THRESHOLD = 10 REMIX_FOLDER_THRESHOLD = 3 SUPPORTED_EXTS = {".flac", ".m4a", ".aac", ".mp3", ".wav", ".ogg"} -DEFAULT_FUZZY_FP_THRESHOLD = 0.1 # allowed fingerprint difference ratio -DEDUP_KEEP_DELTA_THRESHOLD = 50 # score difference to auto-delete duplicates -FINGERPRINT_TIMEOUT = 30 # seconds before fingerprint worker is timed out # ─── Helper: build primary_counts for the entire vault ───────────────────── def build_primary_counts(root_path, progress_callback=None, phase="A", log_callback=None): @@ -149,7 +143,15 @@ def _sanitize_tag_value(value: object, field: str, log_callback=None) -> str: return sanitized -def _ensure_unique_destination(path: str, moves: dict, log_callback=None) -> str: +def _ensure_unique_destination( + path: str, + moves: dict, + log_callback=None, + *, + source_path: str | None = None, + decision_log: list[str] | None = None, +) -> str: + """Resolve destination collisions deterministically by suffixing names.""" if path not in moves.values() and not os.path.exists(path): return path base, ext = os.path.splitext(path) @@ -159,8 +161,14 @@ def _ensure_unique_destination(path: str, moves: dict, log_callback=None) -> str counter += 1 candidate = f"{base} ({counter}){ext}" if log_callback: + source_label = f" for '{os.path.basename(source_path)}'" if source_path else "" log_callback( - f" ! Destination collision detected; renamed to '{os.path.basename(candidate)}'" + f" ! Destination collision{source_label}; renamed to '{os.path.basename(candidate)}'" + ) + if decision_log is not None: + source_label = f" for '{os.path.basename(source_path)}'" if source_path else "" + decision_log.append( + f" ! Destination collision{source_label}; renamed to '{os.path.basename(candidate)}'" ) return candidate @@ -292,73 +300,6 @@ def derive_tags_from_path(filepath: str, music_root: str): return set(parts) -def _parse_fp(fp: str) -> tuple[str, list[int] | bytes] | None: - """Parse fingerprint string into integer list or decoded bytes.""" - text = fp.strip() - if not text: - return None - try: - arr = [int(x) for x in text.replace(',', ' ').split()] - if arr: - return ("ints", arr) - except Exception: - pass - try: - data = base64.urlsafe_b64decode(text + '=' * (-len(text) % 4)) - if data: - return ("bytes", data) - except Exception: - pass - return None - - -def _hamming_ints(a: list[int], b: list[int]) -> float: - n = min(len(a), len(b)) - if n == 0: - return 1.0 - diff = sum(x != y for x, y in zip(a[:n], b[:n])) - return diff / n - - -def _hamming_bytes(a: bytes, b: bytes) -> float: - n = min(len(a), len(b)) - if n == 0: - return 1.0 - diff = 0 - for i in range(n): - diff += (a[i] ^ b[i]).bit_count() - return diff / (n * 8) - - -def fingerprint_distance(fp1: str | None, fp2: str | None) -> float: - """Return normalized Hamming distance between two fingerprint strings.""" - if not fp1 or not fp2: - return 1.0 - if fp1 == fp2: - return 0.0 - p1 = _parse_fp(fp1) - p2 = _parse_fp(fp2) - if not p1 or not p2 or p1[0] != p2[0]: - return 1.0 - kind, a = p1 - _, b = p2 - if kind == "ints": - return _hamming_ints(a, b) # type: ignore[arg-type] - else: - return _hamming_bytes(a, b) # type: ignore[arg-type] - - -def _keep_score(path: str, info: dict, ext_priority: dict) -> float: - """Compute keep score for a file based on extension, metadata and filename.""" - ext = os.path.splitext(path)[1].lower() - pri = ext_priority.get(ext, 99) - ext_score = 1000.0 / (pri + 1) - meta_score = info.get("meta_count", 0) * 10 - fname_score = len(os.path.splitext(os.path.basename(path))[0]) - return ext_score + meta_score + fname_score - - - # ─── A. HELPER: COMPUTE MOVES & TAG INDEX ─────────────────────────────── @@ -375,22 +316,18 @@ def compute_moves_and_tag_index( """ 1) Determine MUSIC_ROOT: if root_path/Music exists, use that; otherwise root_path itself. 2) Scan for all audio files under MUSIC_ROOT. - 3) Deduplicate tracks by fingerprint only and delete lower-priority duplicates. - If ``dry_run`` is True, deduplication actions are simulated only and no - files are modified. - 4) Read metadata into `songs` dict, build: + 3) Read metadata into `songs` dict, build: - album_counts: how many genuine (non-remix) tracks per (artist, album) - remix_counts: how many remix‐tagged tracks per (artist, album) - - cover_counts: how many files share each embedded cover - 5) Phase 4: For each entry in `songs`, decide new folder & filename, + 4) Phase 4: For each entry in `songs`, decide new folder & filename, with special‐case for small “(Remixes)” albums. Returns: - moves: { old_path: new_path, ... } - tag_index: { new_path: { "leftover_tags": [...], "old_paths": [...] }, ... } - decision_log: list of strings explaining each track’s decision - ``coord`` can be provided to collect additional diagnostic data during - near-duplicate detection. + ``enable_phase_c``, ``flush_cache``, ``max_workers``, and ``coord`` are + retained for backward compatibility but are ignored. """ _verify_dependencies() if log_callback is None: @@ -400,8 +337,6 @@ def log_callback(msg): def progress_callback(current, total, message, phase): pass - cfg = load_config() - fuzzy_fp_threshold = float(cfg.get("fuzzy_fp_threshold", DEFAULT_FUZZY_FP_THRESHOLD)) check_cancelled() # ─── 1) Determine MUSIC_ROOT ────────────────────────────────────────────── @@ -411,13 +346,6 @@ def progress_callback(current, total, message, phase): SUPPORTED_EXTS = {".flac", ".m4a", ".aac", ".mp3", ".wav", ".ogg"} - docs_dir = os.path.join(root_path, "Docs") - os.makedirs(docs_dir, exist_ok=True) - db_path = os.path.join(docs_dir, ".soundvault.db") - if flush_cache: - from fingerprint_cache import flush_cache as _flush - _flush(db_path) - # --- Phase 0: Pre-scan entire vault (under MUSIC_ROOT) --- global_counts = build_primary_counts(MUSIC_ROOT, progress_callback, phase="A", log_callback=log_callback) log_callback(f" → Pre-scan: found {len(global_counts)} unique artists") @@ -425,7 +353,7 @@ def progress_callback(current, total, message, phase): log_callback(f" → DEBUG: droeloe count = {global_counts.get('droeloe', 0)}") # --- Phase 1: Scan for audio files --- - log_callback("1/6: Discovering audio files…") + log_callback("1/4: Discovering audio files…") all_audio = [] idx = 0 for dirpath, _, files in os.walk(MUSIC_ROOT): @@ -444,175 +372,21 @@ def progress_callback(current, total, message, phase): total_audio = idx log_callback(f" → Found {total_audio} audio files.") - # ─── Phase 2: Deduplicate using fingerprints ───────────────────────────── - log_callback("2/6: Deduplicating by fingerprint…") - EXT_PRIORITY = {".flac": 0, ".m4a": 1, ".aac": 1, ".mp3": 2, ".wav": 3, ".ogg": 4} - - path_fps: Dict[str, str | None] = {} - from fingerprint_cache import get_fingerprint - - def _compute(path: str) -> tuple[int | None, str | None]: - try: - import acoustid - return acoustid.fingerprint_file(path) - except Exception: - return None, None - - progress_callback(0, len(all_audio), "Fingerprinting", "A") - for idx, p in enumerate(all_audio, start=1): - fp_val = get_fingerprint(p, db_path, _compute) - path_fps[p] = fp_val - progress_callback(idx, len(all_audio), os.path.relpath(p, root_path), "A") - if idx % 50 == 0 or idx == len(all_audio): - log_callback(f" • Fingerprinting {idx}/{len(all_audio)}") - - file_infos: Dict[str, Dict[str, str | None]] = {} - for idx, fullpath in enumerate(all_audio, start=1): - if idx % 50 == 0 or idx == total_audio: - log_callback(f" • Processing file {idx}/{total_audio} for dedupe") - data = get_tags(fullpath) - raw_artist = _normalize_tag_value(data.get("artist"), "artist", log_callback) - if not raw_artist: - raw_artist = os.path.splitext(os.path.basename(fullpath))[0] - raw_artist = collapse_repeats(raw_artist) - title = _normalize_tag_value(data.get("title"), "title", log_callback) - if not title: - title = os.path.splitext(os.path.basename(fullpath))[0] - album = _normalize_tag_value(data.get("album"), "album", log_callback) - year = _normalize_tag_value(data.get("year"), "year", log_callback) - genre = _normalize_tag_value(data.get("genre"), "genre", log_callback) - meta_count = sum( - 1 - for t in [raw_artist, title, album, data.get("track"), year, genre] - if t - ) - primary, _ = extract_primary_and_collabs(raw_artist) - fp = path_fps.get(fullpath) - if isinstance(fp, (bytes, bytearray)): - try: - fp = fp.decode("utf-8") - except Exception: - fp = fp.decode("latin1", errors="ignore") - ext = os.path.splitext(fullpath)[1].lower() - file_infos[fullpath] = { - "primary": primary.lower(), - "title": title.lower(), - "album": album.lower(), - "fp": fp, - "meta_count": meta_count, - "ext": ext, - } - - to_delete: Dict[str, str] = {} - fp_groups: Dict[str, List[str]] = defaultdict(list) - for path, info in file_infos.items(): - if info["fp"]: - fp_groups[info["fp"]].append(path) - - kept_files = set(file_infos.keys()) review_root = os.path.join(root_path, "Manual Review") os.makedirs(review_root, exist_ok=True) - for fp, paths in fp_groups.items(): - if len(paths) <= 1: - continue - scored = sorted(paths, key=lambda p: _keep_score(p, file_infos[p], EXT_PRIORITY), reverse=True) - if len(scored) < 2: - continue - delta = _keep_score(scored[0], file_infos[scored[0]], EXT_PRIORITY) - _keep_score(scored[1], file_infos[scored[1]], EXT_PRIORITY) - if delta >= DEDUP_KEEP_DELTA_THRESHOLD: - for loser in scored[1:]: - if loser not in to_delete: - to_delete[loser] = "Exact FP match" - kept_files.discard(loser) - else: - meta_counts = {p: file_infos[p].get("meta_count", 0) for p in scored} - max_meta = max(meta_counts.values()) - winner = None - for p in scored: - if meta_counts[p] == max_meta: - winner = p - break - for p in scored: - if p == winner: - continue - if p not in to_delete: - to_delete[p] = "Exact FP match" - kept_files.discard(p) - log_callback( - f"Ambiguous duplicates for fingerprint {fp} auto-resolved—kept {os.path.basename(winner)}." - ) - - if coord is not None: - coord.add_exact_dupes([p for p, r in to_delete.items() if r == "Exact FP match"]) - - - # --- Near-duplicate detection ------------------------------------------------- - from near_duplicate_detector import find_near_duplicates - log_callback(" • Detecting near-duplicates…") - cfg = load_config() - mixed_codec_boost = float(cfg.get("mixed_codec_threshold_boost", MIXED_CODEC_THRESHOLD_BOOST)) - near_dupes = find_near_duplicates( - {p: file_infos[p] for p in kept_files}, - EXT_PRIORITY, - fuzzy_fp_threshold, - log_callback, - enable_phase_c, - coord, - max_workers, - mixed_codec_boost=mixed_codec_boost, - ) - if near_dupes.review_required: - log_callback( - f" ! Near-duplicate candidates require review ({len(near_dupes.review_groups)} group(s)); " - "no automatic merges will be applied." - ) - for group in near_dupes.review_groups: - log_callback( - f" - Review {os.path.basename(group.winner)} vs {len(group.losers)} candidates " - f"(max fp distance {group.max_distance:.3f})" - ) - for loser, reason in near_dupes.items(): - if loser not in to_delete: - to_delete[loser] = reason - kept_files.discard(loser) - - - trash_dir = os.path.join(root_path, "Trash") - os.makedirs(trash_dir, exist_ok=True) - for loser, reason in to_delete.items(): - if dry_run: - log_callback( - f" ? (dry-run) Would move duplicate {loser} to Trash ({reason})" - ) - else: - try: - dest = os.path.join(trash_dir, os.path.basename(loser)) - base, ext = os.path.splitext(os.path.basename(loser)) - counter = 1 - while os.path.exists(dest): - dest = os.path.join(trash_dir, f"{base}_{counter}{ext}") - counter += 1 - shutil.move(loser, dest) - log_callback( - f" - Moved duplicate to Trash ({reason}): {loser}" - ) - except Exception as e: - log_callback(f" ! Failed to move {loser} to Trash: {e}") - - # ─── Phase 3: Read metadata & build counters ───────────────────────────────── - log_callback("3/6: Reading metadata and building counters…") + # ─── Phase 2: Read metadata & build counters ───────────────────────────────── + log_callback("2/4: Reading metadata and building counters…") songs = {} album_counts = defaultdict(int) remix_counts = defaultdict(int) - cover_counts = defaultdict(int) - total_kept = len(kept_files) - progress_callback(0, total_kept, "Reading metadata", "B") + total_files = len(all_audio) + progress_callback(0, total_files, "Reading metadata", "B") - for idx, fullpath in enumerate(kept_files, start=1): - if idx % 50 == 0 or idx == total_kept: - log_callback(f" • Reading metadata {idx}/{total_kept}") - progress_callback(idx, total_kept, f"Metadata {os.path.relpath(fullpath, root_path)}", "B") + for idx, fullpath in enumerate(all_audio, start=1): + if idx % 50 == 0 or idx == total_files: + log_callback(f" • Reading metadata {idx}/{total_files}") + progress_callback(idx, total_files, f"Metadata {os.path.relpath(fullpath, root_path)}", "B") data = get_tags(fullpath) raw_artist = _normalize_tag_value(data.get("artist"), "artist", log_callback) @@ -640,18 +414,6 @@ def _compute(path: str) -> tuple[int | None, str | None]: if album and "remix" in album.lower(): remix_counts[(p_lower, album.lower())] += 1 - # 4) EXTRACT EMBEDDED COVER DATA → compute SHA-1 and store first 10 hex digits - cover_hash = None - cover_payloads = [] - try: - _tags, cover_payloads, _error, _reader = read_metadata(fullpath, include_cover=True) - except Exception: - cover_payloads = [] - if cover_payloads: - sha1 = hashlib.sha1(cover_payloads[0]).hexdigest() - cover_hash = sha1[:10] - cover_counts[cover_hash] += 1 - folder_tags = derive_tags_from_path(fullpath, MUSIC_ROOT) songs[fullpath] = { @@ -664,15 +426,14 @@ def _compute(path: str) -> tuple[int | None, str | None]: "genre": genre, "track": track, "part_of_set": part_of_set, - "cover_hash": cover_hash, "folder_tags": folder_tags, "missing_core": not raw_artist or not title, } - log_callback(f" → Collected metadata for {total_kept} files.") + log_callback(f" → Collected metadata for {total_files} files.") # ─── Phase 4: Determine destination for each file (with logging) ───────────── - log_callback("4/6: Determining destination paths for each file…") + log_callback("3/4: Determining destination paths for each file…") tag_index = {} moves = {} decision_log = [] @@ -738,7 +499,13 @@ def _compute(path: str) -> tuple[int | None, str | None]: decision_log.append(f" Renaming to '{new_filename}' (using raw artist)") new_path = os.path.join(base_folder, new_filename) - new_path = _ensure_unique_destination(new_path, moves, log_callback) + new_path = _ensure_unique_destination( + new_path, + moves, + log_callback, + source_path=old_path, + decision_log=decision_log, + ) moves[old_path] = new_path decision_log.append( f" → Final: '{os.path.relpath(new_path, MUSIC_ROOT)}'" @@ -783,7 +550,13 @@ def _compute(path: str) -> tuple[int | None, str | None]: decision_log.append(f" Renaming to '{new_filename}' (using raw artist)") new_path = os.path.join(base_folder, new_filename) - new_path = _ensure_unique_destination(new_path, moves, log_callback) + new_path = _ensure_unique_destination( + new_path, + moves, + log_callback, + source_path=old_path, + decision_log=decision_log, + ) moves[old_path] = new_path decision_log.append( f" → (Remixes folder) placed under By Artist/{main_artist}/{album} " @@ -857,7 +630,13 @@ def _compute(path: str) -> tuple[int | None, str | None]: decision_log.append(f" Renaming to '{new_filename}' (using raw artist)") new_path = os.path.join(base_folder, new_filename) - new_path = _ensure_unique_destination(new_path, moves, log_callback) + new_path = _ensure_unique_destination( + new_path, + moves, + log_callback, + source_path=old_path, + decision_log=decision_log, + ) moves[old_path] = new_path decision_log.append( f" → Final: '{os.path.relpath(new_path, MUSIC_ROOT)}'" @@ -1052,7 +831,7 @@ def log_callback(msg): pass coord=coord, ) - log_callback("5/6: Writing dry-run HTML…") + log_callback("4/4: Writing dry-run HTML…") render_dry_run_html_from_plan(root_path, output_html_path, moves, tag_index, coord=coord) log_callback(f"✓ Dry-run HTML written to: {output_html_path}") @@ -1073,7 +852,7 @@ def apply_indexer_moves( """ 1) Call compute_moves_and_tag_index() to get (moves, tag_index, decision_log). 2) Move/rename each file in `moves`. - 3) Move any leftover non-audio or album cover images into Trash or into the correct folder. + 3) Move any leftover non-audio files into Trash or Docs. 4) Remove empty directories in two passes: first those that held moved files, then a library-wide sweep for any remaining empty folders. Returns summary: {"moved": , "errors": []}. @@ -1087,8 +866,6 @@ def progress_callback(current, total, path, phase): pass check_cancelled() - cfg = load_config() - trim_silence = bool(cfg.get("trim_silence", False)) # Ensure Not Sorted directory exists for user exclusions not_sorted_dir = os.path.join(root_path, "Not Sorted") @@ -1099,23 +876,6 @@ def progress_callback(current, total, path, phase): if not os.path.isdir(music_root): music_root = root_path - - # ─── Phase 0.5: Compute and cache fingerprints ───────────────── - docs_dir = os.path.join(root_path, "Docs") - os.makedirs(docs_dir, exist_ok=True) - db_path = os.path.join(docs_dir, ".soundvault.db") - from fingerprint_generator import compute_fingerprints_parallel - compute_fingerprints_parallel( - root_path, - db_path, - log_callback, - progress_callback, - max_workers, - trim_silence, - phase="A", - cancel_event=cancel_event, - ) - moves, _, _ = compute_moves_and_tag_index( root_path, log_callback, @@ -1239,7 +999,7 @@ def progress_callback(current, total, path, phase): # ─── Phase 7: Generate Playlists with edge-case handling ──────────── try: from playlist_generator import generate_playlists - log_callback("7/7: Generating safe playlists…") + log_callback("4/4: Generating playlists…") def plog(msg): log_callback(msg) @@ -1320,7 +1080,7 @@ def log_callback(msg): pass log_callback(f"✓ Detailed log written to: {log_path}") # Build dry-run HTML - log_callback("5/6: Writing dry-run HTML…") + log_callback("4/4: Writing dry-run HTML…") build_dry_run_html( root_path, output_html_path, @@ -1331,7 +1091,7 @@ def log_callback(msg): pass ) if not dry_run_only: - # Phase 5–6: Actually move audio files and cover images + # Phase 5–6: Actually move audio files actual_summary = apply_indexer_moves( root_path, log_callback, @@ -1351,24 +1111,13 @@ def log_callback(msg): pass def find_duplicates(root_path, log_callback=None): """Return list of (original_path, duplicate_path) that would be marked as - duplicates by the indexer.""" + duplicates by the indexer. - moves, _, _ = compute_moves_and_tag_index( - root_path, - log_callback, - None, - dry_run=True, - enable_phase_c=False, - flush_cache=False, - max_workers=None, - coord=None, - ) - dup_indicator = os.path.join("Duplicates", "") - return [ - (old, new) - for old, new in moves.items() - if dup_indicator in new - ] + Duplicate detection is disabled in the indexer. + """ + if log_callback: + log_callback("Duplicate detection is disabled in the indexer.") + return [] def main(argv: List[str] | None = None) -> None: @@ -1377,9 +1126,9 @@ def main(argv: List[str] | None = None) -> None: parser = argparse.ArgumentParser(description="SoundVault Music Indexer") parser.add_argument("root", help="Library root path") parser.add_argument("--dry-run", action="store_true", help="Preview only") - parser.add_argument("--enable-phase-c", action="store_true", help="Enable cross-album scan") - parser.add_argument("--flush-cache", action="store_true", help="Flush fingerprint cache") - parser.add_argument("--max-workers", type=int, default=None, help="Fingerprint worker count") + parser.add_argument("--enable-phase-c", action="store_true", help="(Ignored) kept for backward compatibility") + parser.add_argument("--flush-cache", action="store_true", help="(Ignored) kept for backward compatibility") + parser.add_argument("--max-workers", type=int, default=None, help="(Ignored) kept for backward compatibility") parser.add_argument("--no-playlists", action="store_true", help="Skip playlist creation") args = parser.parse_args(argv) From cd1f1cd477487edc23b3f48ba0b03ea1af2673db Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 4 Jan 2026 15:37:18 -0500 Subject: [PATCH 279/606] Add keep score helper for duplicate ranking --- music_indexer_api.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/music_indexer_api.py b/music_indexer_api.py index c2a3b1a..b05d6ba 100644 --- a/music_indexer_api.py +++ b/music_indexer_api.py @@ -124,6 +124,16 @@ def sanitize(name: object) -> str: cleaned = "".join(c for c in normalized if c not in invalid).strip() return cleaned or "Unknown" + +def _keep_score(path: str, info: dict, ext_priority: dict) -> float: + """Compute keep score for a file based on extension, metadata and filename.""" + ext = os.path.splitext(path)[1].lower() + pri = ext_priority.get(ext, 99) + ext_score = 1000.0 / (pri + 1) + meta_score = info.get("meta_count", 0) * 10 + fname_score = len(os.path.splitext(os.path.basename(path))[0]) + return ext_score + meta_score + fname_score + def _normalize_tag_value(value: object, field: str, log_callback=None) -> str: text = _coerce_to_string(value).strip() if not text: From 085904b25937fb4f677bac1bbc58f3e3e6962059 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 4 Jan 2026 15:45:19 -0500 Subject: [PATCH 280/606] Remove keep-score ranking from indexer API --- music_indexer_api.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/music_indexer_api.py b/music_indexer_api.py index c2a3b1a..b3dd721 100644 --- a/music_indexer_api.py +++ b/music_indexer_api.py @@ -124,6 +124,8 @@ def sanitize(name: object) -> str: cleaned = "".join(c for c in normalized if c not in invalid).strip() return cleaned or "Unknown" + + def _normalize_tag_value(value: object, field: str, log_callback=None) -> str: text = _coerce_to_string(value).strip() if not text: @@ -1113,7 +1115,9 @@ def find_duplicates(root_path, log_callback=None): """Return list of (original_path, duplicate_path) that would be marked as duplicates by the indexer. - Duplicate detection is disabled in the indexer. + Duplicate detection and keep/winner ranking are intentionally excluded + from the indexer. The indexer treats every file independently and relies + on deterministic routing plus collision handling (suffixing/buckets). """ if log_callback: log_callback("Duplicate detection is disabled in the indexer.") From b9ce8a533b18dc8749c1a34d94021f8c4ca8b4c8 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 4 Jan 2026 15:45:29 -0500 Subject: [PATCH 281/606] Revert "Add `_keep_score` helper to music_indexer_api for duplicate ranking" --- music_indexer_api.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/music_indexer_api.py b/music_indexer_api.py index b05d6ba..c2a3b1a 100644 --- a/music_indexer_api.py +++ b/music_indexer_api.py @@ -124,16 +124,6 @@ def sanitize(name: object) -> str: cleaned = "".join(c for c in normalized if c not in invalid).strip() return cleaned or "Unknown" - -def _keep_score(path: str, info: dict, ext_priority: dict) -> float: - """Compute keep score for a file based on extension, metadata and filename.""" - ext = os.path.splitext(path)[1].lower() - pri = ext_priority.get(ext, 99) - ext_score = 1000.0 / (pri + 1) - meta_score = info.get("meta_count", 0) * 10 - fname_score = len(os.path.splitext(os.path.basename(path))[0]) - return ext_score + meta_score + fname_score - def _normalize_tag_value(value: object, field: str, log_callback=None) -> str: text = _coerce_to_string(value).strip() if not text: From a40ff2c578a7abe596331821f0c1b88e8167279b Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 4 Jan 2026 16:09:12 -0500 Subject: [PATCH 282/606] Decouple duplicate scoring from indexer --- .../indexer_engine/music_indexer_api.py | 267 +----------------- .../indexer_engine/near_duplicate_detector.py | 12 +- near_duplicate_detector.py | 12 +- 3 files changed, 35 insertions(+), 256 deletions(-) diff --git a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py index 4e759ce..c4880bd 100644 --- a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py +++ b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py @@ -4,12 +4,11 @@ import re import shutil # used for relocating special folders import hashlib -import base64 import unicodedata from collections import defaultdict from typing import Dict, List from dry_run_coordinator import DryRunCoordinator -from config import MIXED_CODEC_THRESHOLD_BOOST, load_config +from config import load_config def _verify_dependencies() -> None: """Raise RuntimeError if the real mutagen library is missing.""" @@ -38,9 +37,6 @@ class ID3NoHeaderError(Exception): COMMON_ARTIST_THRESHOLD = 10 REMIX_FOLDER_THRESHOLD = 3 SUPPORTED_EXTS = {".flac", ".m4a", ".aac", ".mp3", ".wav", ".ogg"} -DEFAULT_FUZZY_FP_THRESHOLD = 0.1 # allowed fingerprint difference ratio -DEDUP_KEEP_DELTA_THRESHOLD = 50 # score difference to auto-delete duplicates -FINGERPRINT_TIMEOUT = 30 # seconds before fingerprint worker is timed out # ─── Helper: build primary_counts for the entire vault ───────────────────── def build_primary_counts(root_path, progress_callback=None, phase="A", log_callback=None): @@ -297,74 +293,6 @@ def derive_tags_from_path(filepath: str, music_root: str): return set(parts) -def _parse_fp(fp: str) -> tuple[str, list[int] | bytes] | None: - """Parse fingerprint string into integer list or decoded bytes.""" - text = fp.strip() - if not text: - return None - try: - arr = [int(x) for x in text.replace(',', ' ').split()] - if arr: - return ("ints", arr) - except Exception: - pass - try: - data = base64.urlsafe_b64decode(text + '=' * (-len(text) % 4)) - if data: - return ("bytes", data) - except Exception: - pass - return None - - -def _hamming_ints(a: list[int], b: list[int]) -> float: - n = min(len(a), len(b)) - if n == 0: - return 1.0 - diff = sum(x != y for x, y in zip(a[:n], b[:n])) - return diff / n - - -def _hamming_bytes(a: bytes, b: bytes) -> float: - n = min(len(a), len(b)) - if n == 0: - return 1.0 - diff = 0 - for i in range(n): - diff += (a[i] ^ b[i]).bit_count() - return diff / (n * 8) - - -def fingerprint_distance(fp1: str | None, fp2: str | None) -> float: - """Return normalized Hamming distance between two fingerprint strings.""" - if not fp1 or not fp2: - return 1.0 - if fp1 == fp2: - return 0.0 - p1 = _parse_fp(fp1) - p2 = _parse_fp(fp2) - if not p1 or not p2 or p1[0] != p2[0]: - return 1.0 - kind, a = p1 - _, b = p2 - if kind == "ints": - return _hamming_ints(a, b) # type: ignore[arg-type] - else: - return _hamming_bytes(a, b) # type: ignore[arg-type] - - -def _keep_score(path: str, info: dict, ext_priority: dict) -> float: - """Compute keep score for a file based on extension, metadata and filename.""" - ext = os.path.splitext(path)[1].lower() - pri = ext_priority.get(ext, 99) - ext_score = 1000.0 / (pri + 1) - meta_score = info.get("meta_count", 0) * 10 - fname_score = len(os.path.splitext(os.path.basename(path))[0]) - return ext_score + meta_score + fname_score - - - - # ─── A. HELPER: COMPUTE MOVES & TAG INDEX ─────────────────────────────── def compute_moves_and_tag_index( @@ -380,10 +308,7 @@ def compute_moves_and_tag_index( """ 1) Determine MUSIC_ROOT: if root_path/Music exists, use that; otherwise root_path itself. 2) Scan for all audio files under MUSIC_ROOT. - 3) Deduplicate tracks by fingerprint only and delete lower-priority duplicates. - If ``dry_run`` is True, deduplication actions are simulated only and no - files are modified. - 4) Read metadata into `songs` dict, build: + 3) Read metadata into `songs` dict, build: - album_counts: how many genuine (non-remix) tracks per (artist, album) - remix_counts: how many remix‐tagged tracks per (artist, album) - cover_counts: how many files share each embedded cover @@ -394,8 +319,7 @@ def compute_moves_and_tag_index( - tag_index: { new_path: { "leftover_tags": [...], "old_paths": [...] }, ... } - decision_log: list of strings explaining each track’s decision - ``coord`` can be provided to collect additional diagnostic data during - near-duplicate detection. + ``coord`` can be provided to collect additional diagnostic data. """ _verify_dependencies() if log_callback is None: @@ -405,8 +329,6 @@ def log_callback(msg): def progress_callback(current, total, message, phase): pass - cfg = load_config() - fuzzy_fp_threshold = float(cfg.get("fuzzy_fp_threshold", DEFAULT_FUZZY_FP_THRESHOLD)) check_cancelled() # ─── 1) Determine MUSIC_ROOT ────────────────────────────────────────────── @@ -449,161 +371,11 @@ def progress_callback(current, total, message, phase): total_audio = idx log_callback(f" → Found {total_audio} audio files.") - # ─── Phase 2: Deduplicate using fingerprints ───────────────────────────── - log_callback("2/6: Deduplicating by fingerprint…") - EXT_PRIORITY = {".flac": 0, ".m4a": 1, ".aac": 1, ".mp3": 2, ".wav": 3, ".ogg": 4} - - path_fps: Dict[str, str | None] = {} - from fingerprint_cache import get_fingerprint - - def _compute(path: str) -> tuple[int | None, str | None]: - try: - import acoustid - return acoustid.fingerprint_file(path) - except Exception: - return None, None - - progress_callback(0, len(all_audio), "Fingerprinting", "A") - for idx, p in enumerate(all_audio, start=1): - fp_val = get_fingerprint(p, db_path, _compute) - path_fps[p] = fp_val - progress_callback(idx, len(all_audio), os.path.relpath(p, root_path), "A") - if idx % 50 == 0 or idx == len(all_audio): - log_callback(f" • Fingerprinting {idx}/{len(all_audio)}") - - file_infos: Dict[str, Dict[str, str | None]] = {} - for idx, fullpath in enumerate(all_audio, start=1): - if idx % 50 == 0 or idx == total_audio: - log_callback(f" • Processing file {idx}/{total_audio} for dedupe") - data = get_tags(fullpath) - raw_artist = _normalize_tag_value(data.get("artist"), "artist", log_callback) - if not raw_artist: - raw_artist = os.path.splitext(os.path.basename(fullpath))[0] - raw_artist = collapse_repeats(raw_artist) - title = _normalize_tag_value(data.get("title"), "title", log_callback) - if not title: - title = os.path.splitext(os.path.basename(fullpath))[0] - album = _normalize_tag_value(data.get("album"), "album", log_callback) - year = _normalize_tag_value(data.get("year"), "year", log_callback) - genre = _normalize_tag_value(data.get("genre"), "genre", log_callback) - meta_count = sum( - 1 - for t in [raw_artist, title, album, data.get("track"), year, genre] - if t - ) - primary, _ = extract_primary_and_collabs(raw_artist) - fp = path_fps.get(fullpath) - if isinstance(fp, (bytes, bytearray)): - try: - fp = fp.decode("utf-8") - except Exception: - fp = fp.decode("latin1", errors="ignore") - ext = os.path.splitext(fullpath)[1].lower() - file_infos[fullpath] = { - "primary": primary.lower(), - "title": title.lower(), - "album": album.lower(), - "fp": fp, - "meta_count": meta_count, - "ext": ext, - } - - to_delete: Dict[str, str] = {} - fp_groups: Dict[str, List[str]] = defaultdict(list) - for path, info in file_infos.items(): - if info["fp"]: - fp_groups[info["fp"]].append(path) - - kept_files = set(file_infos.keys()) - review_root = os.path.join(root_path, "Manual Review") - os.makedirs(review_root, exist_ok=True) - - for fp, paths in fp_groups.items(): - if len(paths) <= 1: - continue - scored = sorted(paths, key=lambda p: _keep_score(p, file_infos[p], EXT_PRIORITY), reverse=True) - if len(scored) < 2: - continue - delta = _keep_score(scored[0], file_infos[scored[0]], EXT_PRIORITY) - _keep_score(scored[1], file_infos[scored[1]], EXT_PRIORITY) - if delta >= DEDUP_KEEP_DELTA_THRESHOLD: - for loser in scored[1:]: - if loser not in to_delete: - to_delete[loser] = "Exact FP match" - kept_files.discard(loser) - else: - meta_counts = {p: file_infos[p].get("meta_count", 0) for p in scored} - max_meta = max(meta_counts.values()) - winner = None - for p in scored: - if meta_counts[p] == max_meta: - winner = p - break - for p in scored: - if p == winner: - continue - if p not in to_delete: - to_delete[p] = "Exact FP match" - kept_files.discard(p) - log_callback( - f"Ambiguous duplicates for fingerprint {fp} auto-resolved—kept {os.path.basename(winner)}." - ) - - if coord is not None: - coord.add_exact_dupes([p for p, r in to_delete.items() if r == "Exact FP match"]) - - - # --- Near-duplicate detection ------------------------------------------------- - from near_duplicate_detector import find_near_duplicates - log_callback(" • Detecting near-duplicates…") - cfg = load_config() - mixed_codec_boost = float(cfg.get("mixed_codec_threshold_boost", MIXED_CODEC_THRESHOLD_BOOST)) - near_dupes = find_near_duplicates( - {p: file_infos[p] for p in kept_files}, - EXT_PRIORITY, - fuzzy_fp_threshold, - log_callback, - enable_phase_c, - coord, - max_workers, - mixed_codec_boost=mixed_codec_boost, + # ─── Phase 2: Duplicate detection (disabled in indexer) ─────────────────── + log_callback( + "2/6: Duplicate detection is handled by duplicate tools; skipping in indexer." ) - if near_dupes.review_required: - log_callback( - f" ! Near-duplicate candidates require review ({len(near_dupes.review_groups)} group(s)); " - "no automatic merges will be applied." - ) - for group in near_dupes.review_groups: - log_callback( - f" - Review {os.path.basename(group.winner)} vs {len(group.losers)} candidates " - f"(max fp distance {group.max_distance:.3f})" - ) - for loser, reason in near_dupes.items(): - if loser not in to_delete: - to_delete[loser] = reason - kept_files.discard(loser) - - - trash_dir = os.path.join(root_path, "Trash") - os.makedirs(trash_dir, exist_ok=True) - for loser, reason in to_delete.items(): - if dry_run: - log_callback( - f" ? (dry-run) Would move duplicate {loser} to Trash ({reason})" - ) - else: - try: - dest = os.path.join(trash_dir, os.path.basename(loser)) - base, ext = os.path.splitext(os.path.basename(loser)) - counter = 1 - while os.path.exists(dest): - dest = os.path.join(trash_dir, f"{base}_{counter}{ext}") - counter += 1 - shutil.move(loser, dest) - log_callback( - f" - Moved duplicate to Trash ({reason}): {loser}" - ) - except Exception as e: - log_callback(f" ! Failed to move {loser} to Trash: {e}") + kept_files = set(all_audio) # ─── Phase 3: Read metadata & build counters ───────────────────────────────── log_callback("3/6: Reading metadata and building counters…") @@ -1367,24 +1139,15 @@ def log_callback(msg): pass def find_duplicates(root_path, log_callback=None): """Return list of (original_path, duplicate_path) that would be marked as - duplicates by the indexer.""" + duplicates by the indexer. - moves, _, _ = compute_moves_and_tag_index( - root_path, - log_callback, - None, - dry_run=True, - enable_phase_c=False, - flush_cache=False, - max_workers=None, - coord=None, - ) - dup_indicator = os.path.join("Duplicates", "") - return [ - (old, new) - for old, new in moves.items() - if dup_indicator in new - ] + Duplicate detection and keep/winner ranking are intentionally excluded + from the indexer. The indexer treats every file independently and relies + on deterministic routing plus collision handling (suffixing/buckets). + """ + if log_callback: + log_callback("Duplicate detection is disabled in the indexer.") + return [] def main(argv: List[str] | None = None) -> None: diff --git a/library_sync_indexer_engine/indexer_engine/near_duplicate_detector.py b/library_sync_indexer_engine/indexer_engine/near_duplicate_detector.py index 3901faf..62c09e6 100644 --- a/library_sync_indexer_engine/indexer_engine/near_duplicate_detector.py +++ b/library_sync_indexer_engine/indexer_engine/near_duplicate_detector.py @@ -15,8 +15,6 @@ logger = logging.getLogger(__name__) -from music_indexer_api import _keep_score - EXCLUSION_KEYWORDS = ['remix', 'remastered', 'edit', 'version'] COARSE_FP_BAND_SIZE = 8 COARSE_FP_BANDS = 6 @@ -24,6 +22,16 @@ LOSSLESS_EXTS = {".flac", ".wav", ".alac", ".ape", ".aiff", ".aif"} +def _keep_score(path: str, info: Mapping[str, object], ext_priority: Mapping[str, int]) -> float: + """Compute keep score for a file based on extension, metadata and filename.""" + ext = os.path.splitext(path)[1].lower() + pri = ext_priority.get(ext, 99) + ext_score = 1000.0 / (pri + 1) + meta_score = int(info.get("meta_count", 0) or 0) * 10 + fname_score = len(os.path.splitext(os.path.basename(path))[0]) + return ext_score + meta_score + fname_score + + @dataclass class NearDuplicateGroup: winner: str diff --git a/near_duplicate_detector.py b/near_duplicate_detector.py index 3901faf..62c09e6 100644 --- a/near_duplicate_detector.py +++ b/near_duplicate_detector.py @@ -15,8 +15,6 @@ logger = logging.getLogger(__name__) -from music_indexer_api import _keep_score - EXCLUSION_KEYWORDS = ['remix', 'remastered', 'edit', 'version'] COARSE_FP_BAND_SIZE = 8 COARSE_FP_BANDS = 6 @@ -24,6 +22,16 @@ LOSSLESS_EXTS = {".flac", ".wav", ".alac", ".ape", ".aiff", ".aif"} +def _keep_score(path: str, info: Mapping[str, object], ext_priority: Mapping[str, int]) -> float: + """Compute keep score for a file based on extension, metadata and filename.""" + ext = os.path.splitext(path)[1].lower() + pri = ext_priority.get(ext, 99) + ext_score = 1000.0 / (pri + 1) + meta_score = int(info.get("meta_count", 0) or 0) * 10 + fname_score = len(os.path.splitext(os.path.basename(path))[0]) + return ext_score + meta_score + fname_score + + @dataclass class NearDuplicateGroup: winner: str From f17e89dd357eb471d8a894dee89846384efbc43a Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 4 Jan 2026 20:49:43 -0500 Subject: [PATCH 283/606] Only create disc folders for multi-disc albums --- .../indexer_engine/music_indexer_api.py | 63 +++++++++++++------ music_indexer_api.py | 63 +++++++++++++------ 2 files changed, 90 insertions(+), 36 deletions(-) diff --git a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py index c4880bd..dbca099 100644 --- a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py +++ b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py @@ -193,8 +193,12 @@ def is_repeated(name: str) -> bool: return True return False -def _normalize_part_of_set(raw_value: str | None) -> str | None: - if not raw_value: +def _normalize_disc_value(raw_value: object) -> str | None: + if raw_value is None: + return None + if isinstance(raw_value, tuple): + raw_value = raw_value[0] if raw_value else None + if raw_value is None: return None text = str(raw_value).strip() if not text: @@ -203,20 +207,27 @@ def _normalize_part_of_set(raw_value: str | None) -> str | None: if not first: return None if first.isdigit(): - return f"Part {int(first)}" + return str(int(first)) return first def _apply_part_of_set( base_folder: str, album: str, - part_of_set: str | None, + disc_value: str | None, + is_multi_disc: bool, decision_log: list[str], ) -> str: - if part_of_set and album and os.path.basename(base_folder) == sanitize(album): - part_folder = sanitize(part_of_set) + if is_multi_disc and disc_value and album and os.path.basename(base_folder) == sanitize(album): + if disc_value.lower().startswith("disc "): + disc_folder_label = disc_value + elif disc_value.isdigit(): + disc_folder_label = f"Disc {int(disc_value)}" + else: + disc_folder_label = f"Disc {disc_value}" + part_folder = sanitize(disc_folder_label) decision_log.append( - f" → Part of set '{part_of_set}' detected; placed under subfolder '{part_folder}'" + f" → Multi-disc album; placed under subfolder '{part_folder}'" ) return os.path.join(base_folder, part_folder) return base_folder @@ -224,7 +235,7 @@ def _apply_part_of_set( def get_tags(path: str): """ - Read basic tags using Mutagen (artist, title, album, year, track, genre, part_of_set). + Read basic tags using Mutagen (artist, title, album, year, track, genre, disc_value). Return a dict with those fields (or None if missing). """ tags = read_tags(path) @@ -234,8 +245,8 @@ def get_tags(path: str): year = tags.get("year") track = tags.get("track") genre = tags.get("genre") - raw_part_of_set = tags.get("discnumber") or tags.get("disc") - part_of_set = _normalize_part_of_set(raw_part_of_set) + raw_disc = tags.get("discnumber") or tags.get("disc") + disc_value = _normalize_disc_value(raw_disc) return { "artist": artist, "title": title, @@ -243,7 +254,7 @@ def get_tags(path: str): "year": year, "track": track, "genre": genre, - "part_of_set": part_of_set, + "disc_value": disc_value, } def extract_primary_and_collabs(raw_artist: str): @@ -382,6 +393,7 @@ def progress_callback(current, total, message, phase): songs = {} album_counts = defaultdict(int) remix_counts = defaultdict(int) + album_disc_values = defaultdict(set) cover_counts = defaultdict(int) total_kept = len(kept_files) progress_callback(0, total_kept, "Reading metadata", "B") @@ -403,7 +415,7 @@ def progress_callback(current, total, message, phase): year = _normalize_tag_value(data.get("year"), "year", log_callback) genre = _normalize_tag_value(data.get("genre"), "genre", log_callback) track = data.get("track") - part_of_set = _normalize_tag_value(data.get("part_of_set"), "part_of_set", log_callback) + disc_value = _normalize_tag_value(data.get("disc_value"), "disc_value", log_callback) # 1) Primary & collabs (preserve original case) primary, collabs = extract_primary_and_collabs(raw_artist) @@ -417,6 +429,10 @@ def progress_callback(current, total, message, phase): if album and "remix" in album.lower(): remix_counts[(p_lower, album.lower())] += 1 + # 4) track distinct disc values per (artist, album) + if album and disc_value: + album_disc_values[(p_lower, album.lower())].add(disc_value) + # 4) EXTRACT EMBEDDED COVER DATA → compute SHA-1 and store first 10 hex digits cover_hash = None try: @@ -451,7 +467,7 @@ def progress_callback(current, total, message, phase): "year": year, "genre": genre, "track": track, - "part_of_set": part_of_set, + "disc_value": disc_value, "cover_hash": cover_hash, "folder_tags": folder_tags, "missing_core": not raw_artist or not title, @@ -469,6 +485,9 @@ def progress_callback(current, total, message, phase): # Precompute any necessary lookups from songs data # (album_counts, remix_counts, etc. were built in Phase 3) + multi_disc_albums = { + key for key, disc_values in album_disc_values.items() if len(disc_values) >= 2 + } for old_path, info in songs.items(): index += 1 @@ -480,7 +499,7 @@ def progress_callback(current, total, message, phase): album = info["album"] or "" year = info["year"] or "" track = info["track"] - part_of_set = info.get("part_of_set") + disc_value = info.get("disc_value") folders = info["folder_tags"] year_label = _sanitize_tag_value(year, "year", log_callback) if info.get("missing_core"): @@ -555,7 +574,8 @@ def progress_callback(current, total, message, phase): MUSIC_ROOT, "By Artist", _sanitize_tag_value(main_artist, "artist", log_callback) ) base_folder = os.path.join(artist_folder, _sanitize_tag_value(album, "album", log_callback)) - base_folder = _apply_part_of_set(base_folder, album, part_of_set, decision_log) + is_multi_disc = (p_lower, album.lower()) in multi_disc_albums if album else False + base_folder = _apply_part_of_set(base_folder, album, disc_value, is_multi_disc, decision_log) # Build filename (skip normal album logic) basename = os.path.basename(old_path) @@ -629,7 +649,8 @@ def progress_callback(current, total, message, phase): f"album_count={c2} ≤ 3 → group under '{primary} - Singles'" ) - base_folder = _apply_part_of_set(base_folder, album, part_of_set, decision_log) + is_multi_disc = (p_lower, album.lower()) in multi_disc_albums if album else False + base_folder = _apply_part_of_set(base_folder, album, disc_value, is_multi_disc, decision_log) # Build filename for all “common” tracks basename = os.path.basename(old_path) @@ -656,8 +677,14 @@ def progress_callback(current, total, message, phase): leftover.discard(primary) if album: leftover.discard(album) - if part_of_set: - leftover.discard(part_of_set) + if is_multi_disc and disc_value: + if disc_value.lower().startswith("disc "): + disc_folder_label = disc_value + elif disc_value.isdigit(): + disc_folder_label = f"Disc {int(disc_value)}" + else: + disc_folder_label = f"Disc {disc_value}" + leftover.discard(disc_folder_label) if year: leftover.discard(year) if genre: diff --git a/music_indexer_api.py b/music_indexer_api.py index b3dd721..3a9b174 100644 --- a/music_indexer_api.py +++ b/music_indexer_api.py @@ -202,8 +202,12 @@ def is_repeated(name: str) -> bool: return True return False -def _normalize_part_of_set(raw_value: str | None) -> str | None: - if not raw_value: +def _normalize_disc_value(raw_value: object) -> str | None: + if raw_value is None: + return None + if isinstance(raw_value, tuple): + raw_value = raw_value[0] if raw_value else None + if raw_value is None: return None text = str(raw_value).strip() if not text: @@ -212,20 +216,27 @@ def _normalize_part_of_set(raw_value: str | None) -> str | None: if not first: return None if first.isdigit(): - return f"Part {int(first)}" + return str(int(first)) return first def _apply_part_of_set( base_folder: str, album: str, - part_of_set: str | None, + disc_value: str | None, + is_multi_disc: bool, decision_log: list[str], ) -> str: - if part_of_set and album and os.path.basename(base_folder) == sanitize(album): - part_folder = sanitize(part_of_set) + if is_multi_disc and disc_value and album and os.path.basename(base_folder) == sanitize(album): + if disc_value.lower().startswith("disc "): + disc_folder_label = disc_value + elif disc_value.isdigit(): + disc_folder_label = f"Disc {int(disc_value)}" + else: + disc_folder_label = f"Disc {disc_value}" + part_folder = sanitize(disc_folder_label) decision_log.append( - f" → Part of set '{part_of_set}' detected; placed under subfolder '{part_folder}'" + f" → Multi-disc album; placed under subfolder '{part_folder}'" ) return os.path.join(base_folder, part_folder) return base_folder @@ -233,7 +244,7 @@ def _apply_part_of_set( def get_tags(path: str): """ - Read basic tags using Mutagen (artist, title, album, year, track, genre, part_of_set). + Read basic tags using Mutagen (artist, title, album, year, track, genre, disc_value). Return a dict with those fields (or None if missing). """ tags = read_tags(path) @@ -243,8 +254,8 @@ def get_tags(path: str): year = tags.get("year") track = tags.get("track") genre = tags.get("genre") - raw_part_of_set = tags.get("discnumber") or tags.get("disc") - part_of_set = _normalize_part_of_set(raw_part_of_set) + raw_disc = tags.get("discnumber") or tags.get("disc") + disc_value = _normalize_disc_value(raw_disc) return { "artist": artist, "title": title, @@ -252,7 +263,7 @@ def get_tags(path: str): "year": year, "track": track, "genre": genre, - "part_of_set": part_of_set, + "disc_value": disc_value, } def extract_primary_and_collabs(raw_artist: str): @@ -382,6 +393,7 @@ def progress_callback(current, total, message, phase): songs = {} album_counts = defaultdict(int) remix_counts = defaultdict(int) + album_disc_values = defaultdict(set) total_files = len(all_audio) progress_callback(0, total_files, "Reading metadata", "B") @@ -402,7 +414,7 @@ def progress_callback(current, total, message, phase): year = _normalize_tag_value(data.get("year"), "year", log_callback) genre = _normalize_tag_value(data.get("genre"), "genre", log_callback) track = data.get("track") - part_of_set = _normalize_tag_value(data.get("part_of_set"), "part_of_set", log_callback) + disc_value = _normalize_tag_value(data.get("disc_value"), "disc_value", log_callback) # 1) Primary & collabs (preserve original case) primary, collabs = extract_primary_and_collabs(raw_artist) @@ -416,6 +428,10 @@ def progress_callback(current, total, message, phase): if album and "remix" in album.lower(): remix_counts[(p_lower, album.lower())] += 1 + # 4) track distinct disc values per (artist, album) + if album and disc_value: + album_disc_values[(p_lower, album.lower())].add(disc_value) + folder_tags = derive_tags_from_path(fullpath, MUSIC_ROOT) songs[fullpath] = { @@ -427,7 +443,7 @@ def progress_callback(current, total, message, phase): "year": year, "genre": genre, "track": track, - "part_of_set": part_of_set, + "disc_value": disc_value, "folder_tags": folder_tags, "missing_core": not raw_artist or not title, } @@ -444,6 +460,9 @@ def progress_callback(current, total, message, phase): # Precompute any necessary lookups from songs data # (album_counts, remix_counts, etc. were built in Phase 3) + multi_disc_albums = { + key for key, disc_values in album_disc_values.items() if len(disc_values) >= 2 + } for old_path, info in songs.items(): index += 1 @@ -455,7 +474,7 @@ def progress_callback(current, total, message, phase): album = info["album"] or "" year = info["year"] or "" track = info["track"] - part_of_set = info.get("part_of_set") + disc_value = info.get("disc_value") folders = info["folder_tags"] year_label = _sanitize_tag_value(year, "year", log_callback) if info.get("missing_core"): @@ -536,7 +555,8 @@ def progress_callback(current, total, message, phase): MUSIC_ROOT, "By Artist", _sanitize_tag_value(main_artist, "artist", log_callback) ) base_folder = os.path.join(artist_folder, _sanitize_tag_value(album, "album", log_callback)) - base_folder = _apply_part_of_set(base_folder, album, part_of_set, decision_log) + is_multi_disc = (p_lower, album.lower()) in multi_disc_albums if album else False + base_folder = _apply_part_of_set(base_folder, album, disc_value, is_multi_disc, decision_log) # Build filename (skip normal album logic) basename = os.path.basename(old_path) @@ -616,7 +636,8 @@ def progress_callback(current, total, message, phase): f"album_count={c2} ≤ 3 → group under '{primary} - Singles'" ) - base_folder = _apply_part_of_set(base_folder, album, part_of_set, decision_log) + is_multi_disc = (p_lower, album.lower()) in multi_disc_albums if album else False + base_folder = _apply_part_of_set(base_folder, album, disc_value, is_multi_disc, decision_log) # Build filename for all “common” tracks basename = os.path.basename(old_path) @@ -649,8 +670,14 @@ def progress_callback(current, total, message, phase): leftover.discard(primary) if album: leftover.discard(album) - if part_of_set: - leftover.discard(part_of_set) + if is_multi_disc and disc_value: + if disc_value.lower().startswith("disc "): + disc_folder_label = disc_value + elif disc_value.isdigit(): + disc_folder_label = f"Disc {int(disc_value)}" + else: + disc_folder_label = f"Disc {disc_value}" + leftover.discard(disc_folder_label) if year: leftover.discard(year) if genre: From 92d9947dea318171ae6de868de1dff4d5fc6b2b8 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 4 Jan 2026 21:02:32 -0500 Subject: [PATCH 284/606] Add Year Assistant workflow panel --- main_gui.py | 465 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 465 insertions(+) diff --git a/main_gui.py b/main_gui.py index f5b04c3..c3d5bb3 100644 --- a/main_gui.py +++ b/main_gui.py @@ -127,6 +127,23 @@ logger = logging.getLogger(__name__) +YEAR_ASSISTANT_PROMPT = """ +You will be given a list of music tracks in the format `Artist - Title` (one per line). +For each line, use web search to determine the most likely original release year of the recording +(prefer the earliest official release of the song, not a later remaster unless that is the only +identifiable version). Return ONLY valid JSON in the following exact format: + +[ + {"query":"Artist - Title","year":"YYYY" | null,"confidence":"high|medium|low|unknown","notes":"short source hint"}, + ... +] + +Rules: +- Include one JSON object for every input line and preserve the input order. +- Do not add commentary outside JSON. +- If uncertain or multiple versions exist, set year to null or confidence low and explain in notes. +""" + def make_filters(ex_no_diff: bool, ex_skipped: bool, show_all: bool) -> List[FilterFn]: global _cached_filters @@ -531,6 +548,453 @@ def refresh_panel(): populate_raw_genres(app.raw_genre_list) load_existing_mapping() + frame.refresh_cluster_panel = refresh_panel + refresh_panel() + return frame + elif name == "Year Assistant": + lib_var = tk.StringVar(value=app.library_path or "No library selected") + status_var = tk.StringVar(value="Paste AI results and parse to review changes.") + dry_run_var = tk.BooleanVar(value=True) + app.year_assistant_dry_run_var = dry_run_var + + expected_queries: list[str] = [] + query_to_paths: dict[str, list[str]] = {} + results_by_iid: dict[str, dict[str, Any]] = {} + + def update_controls(): + lib_var.set(app.library_path or "No library selected") + apply_selected_btn.config( + state="normal" if app.library_path else "disabled" + ) + apply_high_btn.config( + state="normal" if app.library_path else "disabled" + ) + + def coerce_entry(item: Any) -> dict[str, str | None] | None: + if isinstance(item, str): + return {"query": item.strip(), "path": None} + if isinstance(item, (list, tuple)): + if len(item) >= 2: + artist = str(item[0]).strip() + title = str(item[1]).strip() + query = f"{artist} - {title}".strip(" -") + path = str(item[2]).strip() if len(item) > 2 and item[2] else None + return {"query": query, "path": path} + if isinstance(item, dict): + query = item.get("query") + artist = item.get("artist") or item.get("Artist") + title = item.get("title") or item.get("Title") + path = ( + item.get("path") + or item.get("filepath") + or item.get("file") + or item.get("filename") + ) + if not query and (artist or title): + artist_str = str(artist or "").strip() + title_str = str(title or "").strip() + query = f"{artist_str} - {title_str}".strip(" -") + if query: + return {"query": str(query).strip(), "path": str(path).strip() if path else None} + return None + + def load_missing_year_entries() -> list[dict[str, str | None]]: + raw_sources = None + for attr in ( + "missing_year_records", + "missing_year_list", + "missing_year_tracks", + "missing_year_entries", + ): + raw_sources = getattr(app, attr, None) + if raw_sources: + break + if isinstance(raw_sources, dict): + raw_sources = list(raw_sources.values()) + if not isinstance(raw_sources, (list, tuple)): + return [] + entries: list[dict[str, str | None]] = [] + for item in raw_sources: + entry = coerce_entry(item) + if entry and entry["query"]: + entries.append(entry) + return entries + + def dedupe_and_sort(lines: list[str]) -> list[str]: + unique: dict[str, None] = {} + for line in lines: + cleaned = line.strip() + if cleaned and cleaned not in unique: + unique[cleaned] = None + return sorted(unique.keys(), key=lambda s: s.casefold()) + + def refresh_track_list(): + nonlocal expected_queries, query_to_paths + entries = load_missing_year_entries() + query_to_paths = {} + for entry in entries: + query = entry["query"] + if not query: + continue + query_to_paths.setdefault(query, []) + if entry.get("path"): + query_to_paths[query].append(str(entry["path"])) + + expected_queries = dedupe_and_sort(list(query_to_paths.keys())) + track_list_box.configure(state="normal") + track_list_box.delete("1.0", "end") + if expected_queries: + track_list_box.insert("1.0", "\n".join(expected_queries)) + else: + track_list_box.insert( + "1.0", + "No missing-year entries available yet.", + ) + track_list_box.configure(state="disabled") + + def copy_instructions(): + app.clipboard_clear() + app.clipboard_append(YEAR_ASSISTANT_PROMPT.strip()) + + def copy_track_list(): + app.clipboard_clear() + app.clipboard_append("\n".join(expected_queries)) + + def set_status(message: str): + status_var.set(message) + + def normalize_year_value(raw_year: Any) -> str | None: + if raw_year is None: + return None + if isinstance(raw_year, int): + raw_year = str(raw_year) + if isinstance(raw_year, str): + cleaned = raw_year.strip() + if re.fullmatch(r"\d{4}", cleaned): + return cleaned + return None + + def parse_results(): + raw_json = results_box.get("1.0", "end").strip() + if not raw_json: + messagebox.showwarning("No Results", "Paste the AI JSON output first.") + return + try: + data = json.loads(raw_json) + except json.JSONDecodeError as exc: + messagebox.showerror("Invalid JSON", str(exc)) + return + if not isinstance(data, list): + messagebox.showerror("Invalid JSON", "Expected a JSON array.") + return + + allowed_confidence = {"high", "medium", "low", "unknown"} + parsed: dict[str, dict[str, Any]] = {} + errors: list[str] = [] + for idx, entry in enumerate(data, start=1): + if not isinstance(entry, dict): + errors.append(f"Entry {idx} must be an object.") + continue + query = entry.get("query") + confidence = entry.get("confidence") + notes = entry.get("notes", "") + year_value = normalize_year_value(entry.get("year")) + + if not isinstance(query, str) or not query.strip(): + errors.append(f"Entry {idx} missing valid query.") + continue + query = query.strip() + if query in parsed: + errors.append(f"Duplicate query found: {query}") + continue + if confidence not in allowed_confidence: + errors.append(f"Entry {idx} has invalid confidence '{confidence}'.") + continue + if not isinstance(notes, str): + notes = str(notes) + + parsed[query] = { + "query": query, + "year": year_value, + "confidence": confidence, + "notes": notes, + } + + if errors: + messagebox.showerror("Parse Errors", "\n".join(errors)) + return + + missing = [q for q in expected_queries if q not in parsed] + extra = [q for q in parsed.keys() if q not in expected_queries] + + results_tree.delete(*results_tree.get_children()) + results_by_iid.clear() + + def add_row(result: dict[str, Any], status: str, notes_suffix: str | None = None): + notes_text = result.get("notes") or "" + if notes_suffix: + notes_text = f"{notes_text} ({notes_suffix})" if notes_text else notes_suffix + values = ( + result.get("query", ""), + result.get("year") or "", + result.get("confidence") or "unknown", + notes_text, + status, + ) + iid = results_tree.insert("", "end", values=values) + results_by_iid[iid] = { + **result, + "status": status, + "notes": notes_text, + } + + for query in expected_queries: + result = parsed.get( + query, + { + "query": query, + "year": None, + "confidence": "unknown", + "notes": "Missing from AI output.", + }, + ) + paths = query_to_paths.get(query, []) + status = "Ready" if result.get("year") else "Unresolved" + notes_suffix = None + if not paths: + status = "Unresolved" + notes_suffix = "No matching files." + elif len(paths) > 1: + notes_suffix = f"Matches {len(paths)} files." + add_row(result, status, notes_suffix) + + for query in extra: + result = parsed[query] + add_row(result, "Skipped", "Query not in input list.") + + message = "Parsed results." + if missing: + message += f" Missing {len(missing)} queries from AI output." + if extra: + message += f" {len(extra)} extra entries skipped." + set_status(message) + + def update_row_status(iid: str, status: str, notes: str | None = None): + row = results_tree.item(iid) + values = list(row.get("values", [])) + if len(values) < 5: + return + values[4] = status + if notes is not None: + values[3] = notes + results_tree.item(iid, values=values) + if iid in results_by_iid: + results_by_iid[iid]["status"] = status + if notes is not None: + results_by_iid[iid]["notes"] = notes + + def write_year_tag(path: str, year: str) -> tuple[bool, str | None, bool]: + try: + audio = MutagenFile(ensure_long_path(path), easy=True) + except Exception as exc: + return False, f"Failed to read {path}: {exc}", False + if audio is None: + return False, f"Unsupported file: {path}", False + old_tags = read_tags(path) + old_year = str(old_tags.get("year") or old_tags.get("date") or "").strip() + if old_year == year: + return True, old_year, False + changed = False + for key in ("date", "year"): + try: + audio[key] = [year] + changed = True + except Exception: + continue + if not changed: + return False, f"Could not write year tags for {path}", False + try: + audio.save() + except Exception as exc: + return False, f"Failed to save {path}: {exc}", False + return True, old_year, True + + def log_audit_entry(path: str, old_year: str | None, new_year: str): + docs_dir = os.path.join(app.library_path, "Docs") + os.makedirs(docs_dir, exist_ok=True) + audit_path = os.path.join(docs_dir, "year_assistant_audit.log") + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + with open(audit_path, "a", encoding="utf-8") as handle: + handle.write( + f"{timestamp} | {path} | old_year={old_year or ''} | new_year={new_year}\n" + ) + + def apply_entries(iids: list[str], *, label: str): + if not app.library_path: + messagebox.showwarning("No Library", "Select a library before applying.") + return + if not iids: + messagebox.showinfo("Apply", "No rows selected.") + return + + dry_run = dry_run_var.get() + applied = 0 + skipped = 0 + errors: list[str] = [] + + for iid in iids: + result = results_by_iid.get(iid) + if not result: + continue + query = result.get("query") + year = result.get("year") + if result.get("status") != "Ready" or not query or not year: + skipped += 1 + update_row_status(iid, "Skipped") + continue + paths = query_to_paths.get(query, []) + if not paths: + skipped += 1 + update_row_status(iid, "Unresolved", "No matching files.") + continue + if dry_run: + applied += len(paths) + continue + for path in paths: + success, info, changed = write_year_tag(path, year) + if not success: + errors.append(info or f"Failed to update {path}") + continue + if not changed: + skipped += 1 + continue + log_audit_entry(path, info, year) + applied += 1 + + summary = f"{label}: updated {applied} files, skipped {skipped}." + if dry_run: + summary = f"{label} (Dry Run): would update {applied} files." + set_status(summary) + if errors: + messagebox.showerror("Apply Errors", "\n".join(errors)) + else: + messagebox.showinfo("Apply Results", summary) + + def apply_selected(): + apply_entries(list(results_tree.selection()), label="Apply Selected") + + def apply_high_confidence(): + high_iids = [ + iid + for iid, result in results_by_iid.items() + if result.get("status") == "Ready" + and result.get("confidence") == "high" + ] + apply_entries(high_iids, label="Apply High Confidence") + + intro = ttk.Frame(frame) + intro.pack(fill="x", padx=10, pady=10) + ttk.Label( + intro, + text=( + "Use the Year Assistant to generate AI instructions for missing year tags, " + "then import AI results to apply release years." + ), + wraplength=520, + justify="left", + ).pack(anchor="w") + + status_row = ttk.Frame(intro) + status_row.pack(fill="x", pady=(6, 0)) + ttk.Label(status_row, text="Library:", width=12).pack(side="left") + ttk.Label(status_row, textvariable=lib_var).pack( + side="left", fill="x", expand=True + ) + + step1_box = ttk.LabelFrame(frame, text="Step 1: Generate AI Instructions") + step1_box.pack(fill="both", expand=False, padx=10, pady=(0, 10)) + + prompt_box = ttk.LabelFrame(step1_box, text="Instructions to paste into the AI") + prompt_box.pack(fill="both", expand=False, padx=8, pady=(8, 6)) + prompt_text = ScrolledText(prompt_box, height=8, wrap="word") + prompt_text.pack(fill="both", expand=True, padx=6, pady=6) + prompt_text.insert("1.0", YEAR_ASSISTANT_PROMPT.strip()) + prompt_text.configure(state="disabled") + ttk.Button(prompt_box, text="Copy AI Instructions", command=copy_instructions).pack( + anchor="e", padx=6, pady=(0, 6) + ) + + track_box = ttk.LabelFrame(step1_box, text="Tracks missing year") + track_box.pack(fill="both", expand=True, padx=8, pady=(0, 8)) + track_list_box = ScrolledText(track_box, height=10, wrap="word") + track_list_box.pack(fill="both", expand=True, padx=6, pady=6) + track_list_box.configure(state="disabled") + ttk.Button(track_box, text="Copy Track List", command=copy_track_list).pack( + anchor="e", padx=6, pady=(0, 6) + ) + + step2_box = ttk.LabelFrame(frame, text="Step 2: Import Results + Apply") + step2_box.pack(fill="both", expand=True, padx=10, pady=(0, 10)) + + ttk.Label( + step2_box, + text="Paste the AI JSON output below and import to review the proposed years.", + wraplength=520, + justify="left", + ).pack(anchor="w", padx=8, pady=(6, 0)) + + results_box = ScrolledText(step2_box, height=8, wrap="word") + results_box.pack(fill="x", expand=False, padx=8, pady=(6, 6)) + + action_bar = ttk.Frame(step2_box) + action_bar.pack(fill="x", padx=8, pady=(0, 6)) + ttk.Button(action_bar, text="Import Results", command=parse_results).pack( + side="left" + ) + ttk.Checkbutton(action_bar, text="Dry Run", variable=dry_run_var).pack( + side="left", padx=(10, 0) + ) + apply_selected_btn = ttk.Button( + action_bar, text="Apply Selected", command=apply_selected + ) + apply_selected_btn.pack(side="right") + apply_high_btn = ttk.Button( + action_bar, text="Apply All High Confidence", command=apply_high_confidence + ) + apply_high_btn.pack(side="right", padx=(0, 6)) + + table_frame = ttk.Frame(step2_box) + table_frame.pack(fill="both", expand=True, padx=8, pady=(0, 6)) + results_scroll = ttk.Scrollbar(table_frame, orient="vertical") + results_scroll.pack(side="right", fill="y") + results_columns = ("Track", "Year", "Confidence", "Notes", "Status") + results_tree = ttk.Treeview( + table_frame, + columns=results_columns, + show="headings", + yscrollcommand=results_scroll.set, + selectmode="extended", + ) + results_scroll.config(command=results_tree.yview) + results_tree.pack(fill="both", expand=True) + + for col in results_columns: + results_tree.heading(col, text=col) + width = 120 + if col == "Track": + width = 240 + elif col == "Notes": + width = 200 + results_tree.column(col, width=width, anchor="w") + + status_label = ttk.Label(step2_box, textvariable=status_var) + status_label.pack(anchor="w", padx=8, pady=(0, 6)) + + def refresh_panel(): + update_controls() + refresh_track_list() + set_status("Ready to import AI results.") + frame.refresh_cluster_panel = refresh_panel refresh_panel() return frame @@ -3849,6 +4313,7 @@ def build_ui(self): "Interactive – KMeans", "Interactive – HDBSCAN", "Genre Normalizer", + "Year Assistant", "Tempo/Energy Buckets", "Auto-DJ", ]: From 07ef7f543a20a722b79d9a52dbbc888b0f1f2f7a Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 4 Jan 2026 21:22:31 -0500 Subject: [PATCH 285/606] Canonicalize artist casing for routing and metadata --- .../indexer_engine/music_indexer_api.py | 130 +++++++++++++++++- music_indexer_api.py | 130 +++++++++++++++++- 2 files changed, 248 insertions(+), 12 deletions(-) diff --git a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py index dbca099..4b88035 100644 --- a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py +++ b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py @@ -39,10 +39,24 @@ class ID3NoHeaderError(Exception): SUPPORTED_EXTS = {".flac", ".m4a", ".aac", ".mp3", ".wav", ".ogg"} # ─── Helper: build primary_counts for the entire vault ───────────────────── +def _select_canonical_primary(casing_counts: dict[str, dict[str, int]]) -> dict[str, str]: + canonical = {} + for normalized, variants in casing_counts.items(): + if not variants: + continue + ordered = sorted( + variants.items(), + key=lambda item: (-item[1], item[0].casefold(), item[0]), + ) + canonical[normalized] = ordered[0][0] + return canonical + + def build_primary_counts(root_path, progress_callback=None, phase="A", log_callback=None): """ Walk the entire vault (By Artist, By Year, Incoming, etc.) and - return a dict mapping each lowercase-normalized artist → total file count. + return a dict mapping each lowercase-normalized artist → total file count, + plus a canonical casing map for each normalized artist. Falls back to filename if metadata “artist” is missing. ``progress_callback`` will be invoked every 100 files with @@ -57,6 +71,7 @@ def log_callback(_m): pass counts = {} + casing_counts = defaultdict(lambda: defaultdict(int)) idx = 0 for dirpath, _, files in os.walk(root_path): rel_dir = os.path.relpath(dirpath, root_path) @@ -89,8 +104,9 @@ def log_callback(_m): primary, _ = extract_primary_and_collabs(raw) p_lower = primary.lower() counts[p_lower] = counts.get(p_lower, 0) + 1 + casing_counts[p_lower][primary] += 1 - return counts + return counts, _select_canonical_primary(casing_counts) # ─── Shared Utility Functions ───────────────────────────────────────── def _coerce_to_string(value: object) -> str: @@ -294,6 +310,78 @@ def extract_primary_and_collabs(raw_artist: str): return (text, []) + +def _split_primary_artist(raw_artist: str) -> tuple[str, str]: + text = _coerce_to_string(raw_artist).strip() + if not text: + return "", "" + + if "/" in text: + idx = text.find("/") + return text[:idx].strip(), text[idx:] + + lowered = text.lower() + separators = [" feat.", " ft.", " & ", " x ", ", ", ";"] + for sep in separators: + idx = lowered.find(sep) + if idx != -1: + return text[:idx].strip(), text[idx:] + + match = re.search(r"(?<=[a-z])(?=[A-Z])", text) + if match: + idx = match.start() + return text[:idx].strip(), text[idx:] + + return text, "" + + +def _apply_canonical_primary(raw_artist: str, canonical_primary: str) -> str | None: + if not canonical_primary: + return None + primary, remainder = _split_primary_artist(raw_artist) + if not primary: + return None + if primary == canonical_primary: + return None + if primary.casefold() != canonical_primary.casefold(): + return None + return f"{canonical_primary}{remainder}" + + +def _update_primary_artist_metadata( + path: str, + canonical_primary: str | None, + log_callback=None, +) -> bool: + if not canonical_primary: + return False + if log_callback is None: + def log_callback(_msg): + pass + + tags = read_tags(path) + raw_artist = tags.get("artist") + updated = _apply_canonical_primary(raw_artist or "", canonical_primary) + if not updated: + return False + audio = MutagenFile(path, easy=True) + if audio is None: + log_callback(f" ! Unable to update artist tag (unsupported format): {path}") + return False + if audio.tags is None and hasattr(audio, "add_tags"): + try: + audio.add_tags() + except Exception: + pass + try: + audio["artist"] = [updated] + audio.save() + log_callback(f" → Updated artist tag to '{updated}' for {os.path.basename(path)}") + return True + except Exception as exc: + log_callback(f" ! Failed to update artist tag for {path}: {exc}") + return False + def derive_tags_from_path(filepath: str, music_root: str): """ If a file lives in subfolders under music_root, gather those folder names @@ -357,7 +445,12 @@ def progress_callback(current, total, message, phase): _flush(db_path) # --- Phase 0: Pre-scan entire vault (under MUSIC_ROOT) --- - global_counts = build_primary_counts(MUSIC_ROOT, progress_callback, phase="A", log_callback=log_callback) + global_counts, canonical_primary_map = build_primary_counts( + MUSIC_ROOT, + progress_callback, + phase="A", + log_callback=log_callback, + ) log_callback(f" → Pre-scan: found {len(global_counts)} unique artists") log_callback(f" → DEBUG: MUSIC_ROOT = {MUSIC_ROOT}") log_callback(f" → DEBUG: droeloe count = {global_counts.get('droeloe', 0)}") @@ -420,6 +513,7 @@ def progress_callback(current, total, message, phase): # 1) Primary & collabs (preserve original case) primary, collabs = extract_primary_and_collabs(raw_artist) p_lower = primary.lower() + canonical_primary = canonical_primary_map.get(p_lower, primary) # 2) album_counts for genuine (non-remix) tracks under each (artist, album) if album and "remix" not in title.lower() and album.strip().lower() != title.strip().lower(): @@ -461,6 +555,7 @@ def progress_callback(current, total, message, phase): songs[fullpath] = { "raw_artist": raw_artist, "primary": primary, + "canonical_primary": canonical_primary, "collabs": collabs, "title": title, "album": album, @@ -495,6 +590,7 @@ def progress_callback(current, total, message, phase): log_callback(f" • Determining destination {index}/{total}") raw_artist = info["raw_artist"] + canonical_primary = info.get("canonical_primary") title = info["title"] or "" album = info["album"] or "" year = info["year"] or "" @@ -510,6 +606,11 @@ def progress_callback(current, total, message, phase): candidate = f"{root_c} ({idx_dup}){ext_c}" idx_dup += 1 moves[old_path] = candidate + tag_index[candidate] = { + "leftover_tags": [], + "old_paths": sorted(folders), + "canonical_primary": canonical_primary, + } decision_log.append( f" → Missing metadata, placed under Manual Review/{os.path.basename(candidate)}" ) @@ -547,6 +648,11 @@ def progress_callback(current, total, message, phase): new_path = os.path.join(base_folder, new_filename) new_path = _ensure_unique_destination(new_path, moves, log_callback) moves[old_path] = new_path + tag_index[new_path] = { + "leftover_tags": [], + "old_paths": sorted(folders), + "canonical_primary": canonical_primary, + } decision_log.append( f" → Final: '{os.path.relpath(new_path, MUSIC_ROOT)}'" ) @@ -565,6 +671,7 @@ def progress_callback(current, total, message, phase): if rcount >= REMIX_FOLDER_THRESHOLD and count_now >= COMMON_ARTIST_THRESHOLD: # Promote primary to remixer (first part of raw_artist before “/”) main_artist = raw_artist.split("/", 1)[0] + main_artist = canonical_primary_map.get(main_artist.lower(), main_artist) p_lower = main_artist.lower() decision_log.append( f" → Enough remixes ({rcount} ≥ {REMIX_FOLDER_THRESHOLD}) " @@ -593,6 +700,11 @@ def progress_callback(current, total, message, phase): new_path = os.path.join(base_folder, new_filename) new_path = _ensure_unique_destination(new_path, moves, log_callback) moves[old_path] = new_path + tag_index[new_path] = { + "leftover_tags": [], + "old_paths": sorted(folders), + "canonical_primary": canonical_primary, + } decision_log.append( f" → (Remixes folder) placed under By Artist/{main_artist}/{album} " f"→ Final: '{os.path.relpath(new_path, MUSIC_ROOT)}'" @@ -610,7 +722,7 @@ def progress_callback(current, total, message, phase): counts = {artist: global_counts.get(artist.lower(), 0) for artist in all_candidates} best_artist = max(counts, key=lambda a: counts[a]) decision_log.append(f" Candidates: {all_candidates}, counts: {counts}") - primary = best_artist + primary = canonical_primary_map.get(best_artist.lower(), best_artist) p_lower = primary.lower() decision_log.append(f" → After ranking, primary = '{primary}'") @@ -695,7 +807,8 @@ def progress_callback(current, total, message, phase): tag_index[new_path] = { "leftover_tags": sanitized_leftover, - "old_paths": sorted(folders) + "old_paths": sorted(folders), + "canonical_primary": canonical_primary, } log_callback(" → Destination paths determined for all files.") @@ -931,7 +1044,7 @@ def progress_callback(current, total, path, phase): cancel_event=cancel_event, ) - moves, _, _ = compute_moves_and_tag_index( + moves, tag_index, _ = compute_moves_and_tag_index( root_path, log_callback, progress_callback, @@ -970,14 +1083,19 @@ def progress_callback(current, total, path, phase): if idx % 50 == 0 or idx == total_moves: log_callback(f" • Moving file {idx}/{total_moves}") os.makedirs(os.path.dirname(new_path), exist_ok=True) + moved_ok = True try: if os.path.abspath(old_path) != os.path.abspath(new_path): shutil.move(old_path, new_path) summary["moved"] += 1 except Exception as e: + moved_ok = False err = f"Failed to move {old_path} → {new_path}: {e}" summary["errors"].append(err) log_callback(f" ! {err}") + if moved_ok: + canonical = tag_index.get(new_path, {}).get("canonical_primary") + _update_primary_artist_metadata(new_path, canonical, log_callback) # Phase 6: Handle non-audio leftovers… docs_dir = os.path.join(root_path, "Docs") diff --git a/music_indexer_api.py b/music_indexer_api.py index 3a9b174..192b6e8 100644 --- a/music_indexer_api.py +++ b/music_indexer_api.py @@ -32,10 +32,24 @@ def MutagenFile(*_a, **_k): SUPPORTED_EXTS = {".flac", ".m4a", ".aac", ".mp3", ".wav", ".ogg"} # ─── Helper: build primary_counts for the entire vault ───────────────────── +def _select_canonical_primary(casing_counts: dict[str, dict[str, int]]) -> dict[str, str]: + canonical = {} + for normalized, variants in casing_counts.items(): + if not variants: + continue + ordered = sorted( + variants.items(), + key=lambda item: (-item[1], item[0].casefold(), item[0]), + ) + canonical[normalized] = ordered[0][0] + return canonical + + def build_primary_counts(root_path, progress_callback=None, phase="A", log_callback=None): """ Walk the entire vault (By Artist, By Year, Incoming, etc.) and - return a dict mapping each lowercase-normalized artist → total file count. + return a dict mapping each lowercase-normalized artist → total file count, + plus a canonical casing map for each normalized artist. Falls back to filename if metadata “artist” is missing. ``progress_callback`` will be invoked every 100 files with @@ -50,6 +64,7 @@ def log_callback(_m): pass counts = {} + casing_counts = defaultdict(lambda: defaultdict(int)) idx = 0 for dirpath, _, files in os.walk(root_path): rel_dir = os.path.relpath(dirpath, root_path) @@ -82,8 +97,9 @@ def log_callback(_m): primary, _ = extract_primary_and_collabs(raw) p_lower = primary.lower() counts[p_lower] = counts.get(p_lower, 0) + 1 + casing_counts[p_lower][primary] += 1 - return counts + return counts, _select_canonical_primary(casing_counts) # ─── Shared Utility Functions ───────────────────────────────────────── def _coerce_to_string(value: object) -> str: @@ -303,6 +319,78 @@ def extract_primary_and_collabs(raw_artist: str): return (text, []) + +def _split_primary_artist(raw_artist: str) -> tuple[str, str]: + text = _coerce_to_string(raw_artist).strip() + if not text: + return "", "" + + if "/" in text: + idx = text.find("/") + return text[:idx].strip(), text[idx:] + + lowered = text.lower() + separators = [" feat.", " ft.", " & ", " x ", ", ", ";"] + for sep in separators: + idx = lowered.find(sep) + if idx != -1: + return text[:idx].strip(), text[idx:] + + match = re.search(r"(?<=[a-z])(?=[A-Z])", text) + if match: + idx = match.start() + return text[:idx].strip(), text[idx:] + + return text, "" + + +def _apply_canonical_primary(raw_artist: str, canonical_primary: str) -> str | None: + if not canonical_primary: + return None + primary, remainder = _split_primary_artist(raw_artist) + if not primary: + return None + if primary == canonical_primary: + return None + if primary.casefold() != canonical_primary.casefold(): + return None + return f"{canonical_primary}{remainder}" + + +def _update_primary_artist_metadata( + path: str, + canonical_primary: str | None, + log_callback=None, +) -> bool: + if not canonical_primary: + return False + if log_callback is None: + def log_callback(_msg): + pass + + tags = read_tags(path) + raw_artist = tags.get("artist") + updated = _apply_canonical_primary(raw_artist or "", canonical_primary) + if not updated: + return False + audio = MutagenFile(path, easy=True) + if audio is None: + log_callback(f" ! Unable to update artist tag (unsupported format): {path}") + return False + if audio.tags is None and hasattr(audio, "add_tags"): + try: + audio.add_tags() + except Exception: + pass + try: + audio["artist"] = [updated] + audio.save() + log_callback(f" → Updated artist tag to '{updated}' for {os.path.basename(path)}") + return True + except Exception as exc: + log_callback(f" ! Failed to update artist tag for {path}: {exc}") + return False + def derive_tags_from_path(filepath: str, music_root: str): """ If a file lives in subfolders under music_root, gather those folder names @@ -360,7 +448,12 @@ def progress_callback(current, total, message, phase): SUPPORTED_EXTS = {".flac", ".m4a", ".aac", ".mp3", ".wav", ".ogg"} # --- Phase 0: Pre-scan entire vault (under MUSIC_ROOT) --- - global_counts = build_primary_counts(MUSIC_ROOT, progress_callback, phase="A", log_callback=log_callback) + global_counts, canonical_primary_map = build_primary_counts( + MUSIC_ROOT, + progress_callback, + phase="A", + log_callback=log_callback, + ) log_callback(f" → Pre-scan: found {len(global_counts)} unique artists") log_callback(f" → DEBUG: MUSIC_ROOT = {MUSIC_ROOT}") log_callback(f" → DEBUG: droeloe count = {global_counts.get('droeloe', 0)}") @@ -419,6 +512,7 @@ def progress_callback(current, total, message, phase): # 1) Primary & collabs (preserve original case) primary, collabs = extract_primary_and_collabs(raw_artist) p_lower = primary.lower() + canonical_primary = canonical_primary_map.get(p_lower, primary) # 2) album_counts for genuine (non-remix) tracks under each (artist, album) if album and "remix" not in title.lower() and album.strip().lower() != title.strip().lower(): @@ -437,6 +531,7 @@ def progress_callback(current, total, message, phase): songs[fullpath] = { "raw_artist": raw_artist, "primary": primary, + "canonical_primary": canonical_primary, "collabs": collabs, "title": title, "album": album, @@ -470,6 +565,7 @@ def progress_callback(current, total, message, phase): log_callback(f" • Determining destination {index}/{total}") raw_artist = info["raw_artist"] + canonical_primary = info.get("canonical_primary") title = info["title"] or "" album = info["album"] or "" year = info["year"] or "" @@ -485,6 +581,11 @@ def progress_callback(current, total, message, phase): candidate = f"{root_c} ({idx_dup}){ext_c}" idx_dup += 1 moves[old_path] = candidate + tag_index[candidate] = { + "leftover_tags": [], + "old_paths": sorted(folders), + "canonical_primary": canonical_primary, + } decision_log.append( f" → Missing metadata, placed under Manual Review/{os.path.basename(candidate)}" ) @@ -528,6 +629,11 @@ def progress_callback(current, total, message, phase): decision_log=decision_log, ) moves[old_path] = new_path + tag_index[new_path] = { + "leftover_tags": [], + "old_paths": sorted(folders), + "canonical_primary": canonical_primary, + } decision_log.append( f" → Final: '{os.path.relpath(new_path, MUSIC_ROOT)}'" ) @@ -546,6 +652,7 @@ def progress_callback(current, total, message, phase): if rcount >= REMIX_FOLDER_THRESHOLD and count_now >= COMMON_ARTIST_THRESHOLD: # Promote primary to remixer (first part of raw_artist before “/”) main_artist = raw_artist.split("/", 1)[0] + main_artist = canonical_primary_map.get(main_artist.lower(), main_artist) p_lower = main_artist.lower() decision_log.append( f" → Enough remixes ({rcount} ≥ {REMIX_FOLDER_THRESHOLD}) " @@ -580,6 +687,11 @@ def progress_callback(current, total, message, phase): decision_log=decision_log, ) moves[old_path] = new_path + tag_index[new_path] = { + "leftover_tags": [], + "old_paths": sorted(folders), + "canonical_primary": canonical_primary, + } decision_log.append( f" → (Remixes folder) placed under By Artist/{main_artist}/{album} " f"→ Final: '{os.path.relpath(new_path, MUSIC_ROOT)}'" @@ -597,7 +709,7 @@ def progress_callback(current, total, message, phase): counts = {artist: global_counts.get(artist.lower(), 0) for artist in all_candidates} best_artist = max(counts, key=lambda a: counts[a]) decision_log.append(f" Candidates: {all_candidates}, counts: {counts}") - primary = best_artist + primary = canonical_primary_map.get(best_artist.lower(), best_artist) p_lower = primary.lower() decision_log.append(f" → After ranking, primary = '{primary}'") @@ -688,7 +800,8 @@ def progress_callback(current, total, message, phase): tag_index[new_path] = { "leftover_tags": sanitized_leftover, - "old_paths": sorted(folders) + "old_paths": sorted(folders), + "canonical_primary": canonical_primary, } log_callback(" → Destination paths determined for all files.") @@ -905,7 +1018,7 @@ def progress_callback(current, total, path, phase): if not os.path.isdir(music_root): music_root = root_path - moves, _, _ = compute_moves_and_tag_index( + moves, tag_index, _ = compute_moves_and_tag_index( root_path, log_callback, progress_callback, @@ -944,14 +1057,19 @@ def progress_callback(current, total, path, phase): if idx % 50 == 0 or idx == total_moves: log_callback(f" • Moving file {idx}/{total_moves}") os.makedirs(os.path.dirname(new_path), exist_ok=True) + moved_ok = True try: if os.path.abspath(old_path) != os.path.abspath(new_path): shutil.move(old_path, new_path) summary["moved"] += 1 except Exception as e: + moved_ok = False err = f"Failed to move {old_path} → {new_path}: {e}" summary["errors"].append(err) log_callback(f" ! {err}") + if moved_ok: + canonical = tag_index.get(new_path, {}).get("canonical_primary") + _update_primary_artist_metadata(new_path, canonical, log_callback) # Phase 6: Handle non-audio leftovers… docs_dir = os.path.join(root_path, "Docs") From 4200de11ce7b66fb1cc0cc43a7af20c69ba6e06e Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 4 Jan 2026 21:31:16 -0500 Subject: [PATCH 286/606] Use canonical primary casing for artist folders --- .../indexer_engine/music_indexer_api.py | 14 ++++++++++---- music_indexer_api.py | 14 ++++++++++---- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py index 4b88035..87e75e8 100644 --- a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py +++ b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py @@ -677,8 +677,11 @@ def progress_callback(current, total, message, phase): f" → Enough remixes ({rcount} ≥ {REMIX_FOLDER_THRESHOLD}) " f"AND count_now ({count_now}) ≥ {COMMON_ARTIST_THRESHOLD}; force primary = '{main_artist}'" ) + folder_primary = main_artist + if canonical_primary and canonical_primary.casefold() == main_artist.casefold(): + folder_primary = canonical_primary artist_folder = os.path.join( - MUSIC_ROOT, "By Artist", _sanitize_tag_value(main_artist, "artist", log_callback) + MUSIC_ROOT, "By Artist", _sanitize_tag_value(folder_primary, "artist", log_callback) ) base_folder = os.path.join(artist_folder, _sanitize_tag_value(album, "album", log_callback)) is_multi_disc = (p_lower, album.lower()) in multi_disc_albums if album else False @@ -727,15 +730,18 @@ def progress_callback(current, total, message, phase): decision_log.append(f" → After ranking, primary = '{primary}'") # Build “By Artist/” base path + folder_primary = primary + if canonical_primary and canonical_primary.casefold() == primary.casefold(): + folder_primary = canonical_primary artist_folder = os.path.join( - MUSIC_ROOT, "By Artist", _sanitize_tag_value(primary, "artist", log_callback) + MUSIC_ROOT, "By Artist", _sanitize_tag_value(folder_primary, "artist", log_callback) ) # ─── 4.4) COMMON ARTIST: decide between “Album” vs. “Singles” ─────────────── # (At this point, count_now ≥ COMMON_ARTIST_THRESHOLD is guaranteed.) if not album or album.strip().lower() == title.strip().lower(): # No album tag (or album == title) → put into “ - Singles” - primary_label = _sanitize_tag_value(primary, "artist", log_callback) + primary_label = _sanitize_tag_value(folder_primary, "artist", log_callback) base_folder = os.path.join(artist_folder, f"{primary_label} - Singles") decision_log.append( f" Count {count_now} ≥ {COMMON_ARTIST_THRESHOLD}, no album or album=title " @@ -754,7 +760,7 @@ def progress_callback(current, total, message, phase): ) else: # Few tracks in album → treat as a “Singles” release - primary_label = _sanitize_tag_value(primary, "artist", log_callback) + primary_label = _sanitize_tag_value(folder_primary, "artist", log_callback) base_folder = os.path.join(artist_folder, f"{primary_label} - Singles") decision_log.append( f" Count {count_now} ≥ {COMMON_ARTIST_THRESHOLD}, but " diff --git a/music_indexer_api.py b/music_indexer_api.py index 192b6e8..aad1a03 100644 --- a/music_indexer_api.py +++ b/music_indexer_api.py @@ -658,8 +658,11 @@ def progress_callback(current, total, message, phase): f" → Enough remixes ({rcount} ≥ {REMIX_FOLDER_THRESHOLD}) " f"AND count_now ({count_now}) ≥ {COMMON_ARTIST_THRESHOLD}; force primary = '{main_artist}'" ) + folder_primary = main_artist + if canonical_primary and canonical_primary.casefold() == main_artist.casefold(): + folder_primary = canonical_primary artist_folder = os.path.join( - MUSIC_ROOT, "By Artist", _sanitize_tag_value(main_artist, "artist", log_callback) + MUSIC_ROOT, "By Artist", _sanitize_tag_value(folder_primary, "artist", log_callback) ) base_folder = os.path.join(artist_folder, _sanitize_tag_value(album, "album", log_callback)) is_multi_disc = (p_lower, album.lower()) in multi_disc_albums if album else False @@ -714,15 +717,18 @@ def progress_callback(current, total, message, phase): decision_log.append(f" → After ranking, primary = '{primary}'") # Build “By Artist/” base path + folder_primary = primary + if canonical_primary and canonical_primary.casefold() == primary.casefold(): + folder_primary = canonical_primary artist_folder = os.path.join( - MUSIC_ROOT, "By Artist", _sanitize_tag_value(primary, "artist", log_callback) + MUSIC_ROOT, "By Artist", _sanitize_tag_value(folder_primary, "artist", log_callback) ) # ─── 4.4) COMMON ARTIST: decide between “Album” vs. “Singles” ─────────────── # (At this point, count_now ≥ COMMON_ARTIST_THRESHOLD is guaranteed.) if not album or album.strip().lower() == title.strip().lower(): # No album tag (or album == title) → put into “ - Singles” - primary_label = _sanitize_tag_value(primary, "artist", log_callback) + primary_label = _sanitize_tag_value(folder_primary, "artist", log_callback) base_folder = os.path.join(artist_folder, f"{primary_label} - Singles") decision_log.append( f" Count {count_now} ≥ {COMMON_ARTIST_THRESHOLD}, no album or album=title " @@ -741,7 +747,7 @@ def progress_callback(current, total, message, phase): ) else: # Few tracks in album → treat as a “Singles” release - primary_label = _sanitize_tag_value(primary, "artist", log_callback) + primary_label = _sanitize_tag_value(folder_primary, "artist", log_callback) base_folder = os.path.join(artist_folder, f"{primary_label} - Singles") decision_log.append( f" Count {count_now} ≥ {COMMON_ARTIST_THRESHOLD}, but " From 26b51945a2ffc5f4a726493dca748289c348d3bb Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 4 Jan 2026 22:00:51 -0500 Subject: [PATCH 287/606] Add trailing numeric suffix cleanup --- .../indexer_engine/music_indexer_api.py | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py index 87e75e8..f44cea4 100644 --- a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py +++ b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py @@ -181,6 +181,40 @@ def _ensure_unique_destination(path: str, moves: dict, log_callback=None) -> str ) return candidate + +def _cleanup_trailing_numeric_suffixes( + moves: dict[str, str], + tag_index: dict[str, dict], + log_callback=None, +) -> None: + if log_callback is None: + def log_callback(_msg): + pass + + occupied = set(moves.values()) + suffix_pattern = re.compile(r"^(?P.+) \((?P\d+)\)$") + for old_path, new_path in list(moves.items()): + dir_name, filename = os.path.split(new_path) + stem, ext = os.path.splitext(filename) + match = suffix_pattern.match(stem) + if not match: + continue + candidate_filename = f"{match.group('base')}{ext}" + candidate_path = os.path.join(dir_name, candidate_filename) + if candidate_path == new_path: + continue + if candidate_path in occupied or os.path.exists(candidate_path): + continue + occupied.remove(new_path) + occupied.add(candidate_path) + moves[old_path] = candidate_path + if new_path in tag_index: + tag_index[candidate_path] = tag_index.pop(new_path) + log_callback( + f" • Removed redundant suffix: '{os.path.basename(new_path)}' → " + f"'{os.path.basename(candidate_path)}'" + ) + def collapse_repeats(name: str) -> str: """ If name is something like 'DROELOEDROELOE', collapse to 'DROELOE'. @@ -1060,6 +1094,7 @@ def progress_callback(current, total, path, phase): max_workers=max_workers, coord=None, ) + _cleanup_trailing_numeric_suffixes(moves, tag_index, log_callback) From 13a7da5c69ba28e4657f875538b1cfc325dbe1bf Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 4 Jan 2026 22:12:29 -0500 Subject: [PATCH 288/606] Align dry-run preview with suffix cleanup --- .../indexer_engine/music_indexer_api.py | 39 +++++++++- music_indexer_api.py | 74 ++++++++++++++++++- 2 files changed, 111 insertions(+), 2 deletions(-) diff --git a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py index f44cea4..645681b 100644 --- a/library_sync_indexer_engine/indexer_engine/music_indexer_api.py +++ b/library_sync_indexer_engine/indexer_engine/music_indexer_api.py @@ -215,6 +215,36 @@ def log_callback(_msg): f"'{os.path.basename(candidate_path)}'" ) + +def _preview_trailing_suffix_cleanup( + moves: dict[str, str], + tag_index: dict[str, dict], +) -> tuple[dict[str, str], dict[str, dict]]: + preview_moves = dict(moves) + preview_tag_index = dict(tag_index) + if not preview_moves: + return preview_moves, preview_tag_index + occupied = set(preview_moves.values()) + suffix_pattern = re.compile(r"^(?P.+) \((?P\d+)\)$") + for old_path, new_path in list(preview_moves.items()): + dir_name, filename = os.path.split(new_path) + stem, ext = os.path.splitext(filename) + match = suffix_pattern.match(stem) + if not match: + continue + candidate_filename = f"{match.group('base')}{ext}" + candidate_path = os.path.join(dir_name, candidate_filename) + if candidate_path == new_path: + continue + if candidate_path in occupied or os.path.exists(candidate_path): + continue + occupied.remove(new_path) + occupied.add(candidate_path) + preview_moves[old_path] = candidate_path + if new_path in preview_tag_index: + preview_tag_index[candidate_path] = preview_tag_index.pop(new_path) + return preview_moves, preview_tag_index + def collapse_repeats(name: str) -> str: """ If name is something like 'DROELOEDROELOE', collapse to 'DROELOE'. @@ -1022,7 +1052,14 @@ def log_callback(msg): pass log_callback("5/6: Writing dry-run HTML…") - render_dry_run_html_from_plan(root_path, output_html_path, moves, tag_index, coord=coord) + preview_moves, preview_tag_index = _preview_trailing_suffix_cleanup(moves, tag_index) + render_dry_run_html_from_plan( + root_path, + output_html_path, + preview_moves, + preview_tag_index, + coord=coord, + ) log_callback(f"✓ Dry-run HTML written to: {output_html_path}") return {"moved": 0, "html": output_html_path, "dry_run": True} diff --git a/music_indexer_api.py b/music_indexer_api.py index aad1a03..3a15307 100644 --- a/music_indexer_api.py +++ b/music_indexer_api.py @@ -190,6 +190,70 @@ def _ensure_unique_destination( ) return candidate + +def _cleanup_trailing_numeric_suffixes( + moves: dict[str, str], + tag_index: dict[str, dict], + log_callback=None, +) -> None: + if log_callback is None: + def log_callback(_msg): + pass + + occupied = set(moves.values()) + suffix_pattern = re.compile(r"^(?P.+) \((?P\d+)\)$") + for old_path, new_path in list(moves.items()): + dir_name, filename = os.path.split(new_path) + stem, ext = os.path.splitext(filename) + match = suffix_pattern.match(stem) + if not match: + continue + candidate_filename = f"{match.group('base')}{ext}" + candidate_path = os.path.join(dir_name, candidate_filename) + if candidate_path == new_path: + continue + if candidate_path in occupied or os.path.exists(candidate_path): + continue + occupied.remove(new_path) + occupied.add(candidate_path) + moves[old_path] = candidate_path + if new_path in tag_index: + tag_index[candidate_path] = tag_index.pop(new_path) + log_callback( + f" • Removed redundant suffix: '{os.path.basename(new_path)}' → " + f"'{os.path.basename(candidate_path)}'" + ) + + +def _preview_trailing_suffix_cleanup( + moves: dict[str, str], + tag_index: dict[str, dict], +) -> tuple[dict[str, str], dict[str, dict]]: + preview_moves = dict(moves) + preview_tag_index = dict(tag_index) + if not preview_moves: + return preview_moves, preview_tag_index + occupied = set(preview_moves.values()) + suffix_pattern = re.compile(r"^(?P.+) \((?P\d+)\)$") + for old_path, new_path in list(preview_moves.items()): + dir_name, filename = os.path.split(new_path) + stem, ext = os.path.splitext(filename) + match = suffix_pattern.match(stem) + if not match: + continue + candidate_filename = f"{match.group('base')}{ext}" + candidate_path = os.path.join(dir_name, candidate_filename) + if candidate_path == new_path: + continue + if candidate_path in occupied or os.path.exists(candidate_path): + continue + occupied.remove(new_path) + occupied.add(candidate_path) + preview_moves[old_path] = candidate_path + if new_path in preview_tag_index: + preview_tag_index[candidate_path] = preview_tag_index.pop(new_path) + return preview_moves, preview_tag_index + def collapse_repeats(name: str) -> str: """ If name is something like 'DROELOEDROELOE', collapse to 'DROELOE'. @@ -981,7 +1045,14 @@ def log_callback(msg): pass log_callback("4/4: Writing dry-run HTML…") - render_dry_run_html_from_plan(root_path, output_html_path, moves, tag_index, coord=coord) + preview_moves, preview_tag_index = _preview_trailing_suffix_cleanup(moves, tag_index) + render_dry_run_html_from_plan( + root_path, + output_html_path, + preview_moves, + preview_tag_index, + coord=coord, + ) log_callback(f"✓ Dry-run HTML written to: {output_html_path}") return {"moved": 0, "html": output_html_path, "dry_run": True} @@ -1035,6 +1106,7 @@ def progress_callback(current, total, path, phase): coord=None, ) + _cleanup_trailing_numeric_suffixes(moves, tag_index, log_callback) total_moves = len(moves) From a6f0b7d4da8f94c3ad191fb7b44d51af798a11f3 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 4 Jan 2026 22:32:46 -0500 Subject: [PATCH 289/606] Tighten artwork similarity threshold --- duplicate_consolidation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index 6351b94..a3dd9d7 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -49,7 +49,7 @@ EXACT_DUPLICATE_THRESHOLD = 0.02 NEAR_DUPLICATE_THRESHOLD = 0.10 ARTWORK_HASH_SIZE = 8 -ARTWORK_SIMILARITY_THRESHOLD = 10 +ARTWORK_SIMILARITY_THRESHOLD = 5 ARTWORK_VASTLY_DIFFERENT_THRESHOLD = 24 DEFAULT_MAX_CANDIDATES = 5000 DEFAULT_MAX_COMPARISONS = 50_000 From 28fd0e5e28368c90d9d583162e7c3910d772ecd0 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Sun, 4 Jan 2026 22:45:52 -0500 Subject: [PATCH 290/606] Improve duplicate finder progress feedback --- main_gui.py | 85 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 81 insertions(+), 4 deletions(-) diff --git a/main_gui.py b/main_gui.py index c3d5bb3..59859e5 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1960,6 +1960,7 @@ def __init__(self, parent: tk.Widget, library_path: str): self.resizable(True, True) self.status_var = tk.StringVar(value="Idle") + self.fingerprint_status_var = tk.StringVar(value="Ready.") self.progress_var = tk.DoubleVar(value=0) self.library_path_var = tk.StringVar(value=library_path or "") self.playlist_path_var = tk.StringVar( @@ -1980,6 +1981,11 @@ def __init__(self, parent: tk.Widget, library_path: str): self.preview_html_path: str | None = None self.execution_report_path: str | None = None self._plan = None + self._progress_weights = { + "fingerprinting": 0.7, + "grouping": 0.2, + "preview": 0.1, + } container = ttk.Frame(self) container.pack(fill="both", expand=True, padx=10, pady=10) @@ -2098,6 +2104,12 @@ def __init__(self, parent: tk.Widget, library_path: str): groups_frame = ttk.LabelFrame(results, text="Duplicate Groups") groups_frame.grid(row=0, column=0, sticky="nsew", padx=(6, 3), pady=6) + ttk.Label( + groups_frame, + textvariable=self.fingerprint_status_var, + anchor="w", + foreground="#555", + ).pack(fill="x", padx=6, pady=(6, 0)) cols = ("group", "title", "count", "status") self.groups_tree = ttk.Treeview( groups_frame, @@ -2142,6 +2154,7 @@ def _clear_plan(self, note: str | None = None) -> None: self.group_details.insert("end", "Generate a plan to view duplicate groups.") self.group_details.configure(state="disabled") self._reset_group_selection() + self._set_fingerprint_status("Ready.") if note: self._log_action(note) @@ -2369,6 +2382,47 @@ def _set_status(self, status: str, progress: float | None = None) -> None: if progress is not None: self.progress_var.set(progress) + def _set_fingerprint_status(self, status: str) -> None: + self.fingerprint_status_var.set(status) + + def _weighted_progress(self, phase: str, ratio: float) -> float: + order = ("fingerprinting", "grouping", "preview") + total = 0.0 + for step in order: + weight = self._progress_weights.get(step, 0.0) + if step == phase: + clamped = max(0.0, min(1.0, ratio)) + total += weight * clamped + break + total += weight + return max(0.0, min(100.0, total * 100.0)) + + def _format_track_label(self, path: str, audio: object | None) -> str: + if not audio: + return os.path.basename(path) + tags = getattr(audio, "tags", None) or {} + + def _get_tag(keys: tuple[str, ...]) -> str | None: + for key in keys: + if key in tags: + raw = tags[key] + if hasattr(raw, "text"): + raw = raw.text + if isinstance(raw, (list, tuple)): + if raw: + return str(raw[0]) + elif raw is not None: + return str(raw) + return None + + title = _get_tag(("title", "TIT2", "\xa9nam")) + artist = _get_tag(("artist", "TPE1", "\xa9ART")) + if artist and title: + return f"{artist} - {title}" + if title: + return title + return os.path.basename(path) + def _validate_library_root(self) -> str | None: path = self.library_path_var.get().strip() if not path: @@ -2614,7 +2668,16 @@ def _gather_tracks(self, library_root: str) -> list[dict[str, object]]: audio_paths.append(os.path.join(dirpath, fname)) tracks: list[dict[str, object]] = [] - total = len(audio_paths) or 1 + total = len(audio_paths) + if total == 0: + self._set_fingerprint_status("No eligible audio files found.") + self._set_status("Idle", progress=0) + return tracks + self._set_fingerprint_status(f"Fingerprinting 0/{total}") + self._set_status("Fingerprinting…", progress=self._weighted_progress("fingerprinting", 0)) + self.update_idletasks() + last_update = time.monotonic() + update_interval = 0.2 for idx, path in enumerate(sorted(audio_paths), start=1): fingerprint_trace: dict[str, object] = {} fp = get_fingerprint( @@ -2629,6 +2692,7 @@ def _gather_tracks(self, library_root: str) -> list[dict[str, object]]: bitrate = 0 sample_rate = 0 bit_depth = 0 + audio = None try: audio = MutagenFile(path) info = getattr(audio, "info", None) @@ -2638,6 +2702,16 @@ def _gather_tracks(self, library_root: str) -> list[dict[str, object]]: bit_depth = int(getattr(info, "bits_per_sample", 0) or getattr(info, "bitdepth", 0) or 0) except Exception: pass + now = time.monotonic() + if idx == total or now - last_update >= update_interval: + label = self._format_track_label(path, audio) + self._set_fingerprint_status(f"Fingerprinting {idx}/{total} — {label}") + self._set_status( + "Fingerprinting…", + progress=self._weighted_progress("fingerprinting", idx / total), + ) + self.update_idletasks() + last_update = now tracks.append( { "path": path, @@ -2654,7 +2728,6 @@ def _gather_tracks(self, library_root: str) -> list[dict[str, object]]: }, } ) - self._set_status("Scanning…", progress=min(90, int(idx / total * 70) + 20)) return tracks def _generate_plan(self, write_preview: bool) -> None: @@ -2667,7 +2740,8 @@ def _generate_plan(self, write_preview: bool) -> None: messagebox.showwarning("No Tracks", "No audio tracks were found in the selected library.") self._set_status("Idle", progress=0) return - self._set_status("Building plan…", progress=25) + self._set_fingerprint_status("Grouping duplicates…") + self._set_status("Grouping…", progress=self._weighted_progress("grouping", 0)) cfg = load_config() threshold_settings = self._current_threshold_settings() exact_threshold = threshold_settings["exact_duplicate_threshold"] @@ -2687,6 +2761,7 @@ def _generate_plan(self, write_preview: bool) -> None: fingerprint_settings=fingerprint_settings, threshold_settings=threshold_settings, ) + self._set_status("Grouping…", progress=self._weighted_progress("grouping", 1)) self._plan = plan self.group_disposition_overrides.clear() self._apply_deletion_mode() @@ -2694,6 +2769,8 @@ def _generate_plan(self, write_preview: bool) -> None: docs_dir = os.path.join(path, "Docs") os.makedirs(docs_dir, exist_ok=True) if write_preview: + self._set_fingerprint_status("Generating preview…") + self._set_status("Preview…", progress=self._weighted_progress("preview", 0)) json_path = os.path.join(docs_dir, "duplicate_preview.json") export_consolidation_preview(plan, json_path) self.preview_json_path = json_path @@ -2706,7 +2783,7 @@ def _generate_plan(self, write_preview: bool) -> None: ) self.preview_html_path = html_path self._log_action(f"Preview HTML written to {html_path}") - self._set_status("Preview generated", progress=100) + self._set_status("Preview generated", progress=self._weighted_progress("preview", 1)) else: self.preview_json_path = None self.preview_html_path = None From dd6d952de018a2c9cec67a5a6d5019bfd44ee97c Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 5 Jan 2026 20:34:19 -0500 Subject: [PATCH 291/606] Verify execution outcomes in consolidation reports --- duplicate_consolidation_executor.py | 338 +++++++++++++++++++++++++--- main_gui.py | 28 ++- 2 files changed, 335 insertions(+), 31 deletions(-) diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index 9e89b6e..c2dc2ae 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -93,6 +93,10 @@ class ExecutionAction: target: str status: str detail: str + planned: bool = False + attempted: bool = False + verified: bool = False + verification_detail: str | None = None metadata: Dict[str, object] = field(default_factory=dict) def to_dict(self) -> Dict[str, object]: @@ -101,6 +105,10 @@ def to_dict(self) -> Dict[str, object]: "target": self.target, "status": self.status, "detail": self.detail, + "planned": self.planned, + "attempted": self.attempted, + "verified": self.verified, + "verification_detail": self.verification_detail, "metadata": dict(self.metadata), } @@ -236,6 +244,17 @@ def _write_playlist_atomic(path: str, lines: Sequence[str]) -> None: pass +def _verify_playlist_write(path: str, expected_lines: Sequence[str]) -> tuple[bool, str]: + try: + with open(path, "r", encoding="utf-8") as handle: + actual_lines = [ln.rstrip("\n") for ln in handle] + except FileNotFoundError: + return False, "Playlist missing after rewrite." + if list(expected_lines) != actual_lines: + return False, "Playlist contents do not match expected rewrite." + return True, "Playlist rewrite verified." + + def _validate_playlist(path: str, *, base_dir: str | None = None) -> List[str]: playlist_dir = base_dir or os.path.dirname(path) try: @@ -373,13 +392,27 @@ def _apply_artwork(action: ArtworkDirective) -> tuple[bool, str]: _close_mutagen_audio(audio) -def _apply_metadata(target: str, planned_tags: Mapping[str, object]) -> tuple[bool, str]: +def _verify_artwork_write(action: ArtworkDirective) -> tuple[bool, str]: + if not os.path.exists(action.target): + return False, "Target missing after artwork update." + sidecar = f"{action.target}.artwork" + if not os.path.exists(sidecar): + return False, "Artwork sidecar missing after update." + try: + if os.path.getsize(sidecar) <= 0: + return False, "Artwork sidecar is empty." + except OSError: + return False, "Artwork sidecar could not be read." + return True, "Artwork write verified." + + +def _apply_metadata(target: str, planned_tags: Mapping[str, object]) -> tuple[bool, str, List[str]]: if MutagenFile is None: - return True, "Mutagen unavailable; metadata not written." + return True, "Mutagen unavailable; metadata not written.", [] audio = MutagenFile(ensure_long_path(target), easy=True) try: if audio is None: - return True, "Unsupported audio format; metadata not written." + return True, "Unsupported audio format; metadata not written.", [] changed = False skipped: List[str] = [] for key, value in planned_tags.items(): @@ -400,11 +433,36 @@ def _apply_metadata(target: str, planned_tags: Mapping[str, object]) -> tuple[bo try: audio.save() except Exception: - return False, "Failed to save metadata tags." + return False, "Failed to save metadata tags.", skipped if skipped: skipped_list = ", ".join(sorted(set(skipped))) - return True, f"Metadata normalized; skipped unsupported keys: {skipped_list}." - return True, "Metadata normalized." + return True, f"Metadata normalized; skipped unsupported keys: {skipped_list}.", skipped + return True, "Metadata normalized.", skipped + finally: + _close_mutagen_audio(audio) + + +def _verify_metadata_write(target: str, planned_tags: Mapping[str, object], skipped_keys: Iterable[str]) -> tuple[bool, str]: + if not os.path.exists(target): + return False, "Target missing after metadata update." + if MutagenFile is None: + return False, "Mutagen unavailable; metadata not verified." + audio = MutagenFile(ensure_long_path(target), easy=True) + try: + if audio is None or audio.tags is None: + return False, "Metadata unavailable after update." + skipped = set(skipped_keys) + mismatched: List[str] = [] + for key, value in planned_tags.items(): + if value is None or key in INTERNAL_TAG_KEYS or key in skipped: + continue + normalized = [str(value)] if not isinstance(value, list) else [str(v) for v in value] + current = audio.tags.get(key) + if current != normalized: + mismatched.append(key) + if mismatched: + return False, f"Metadata mismatch after write: {', '.join(sorted(mismatched))}." + return True, "Metadata write verified." finally: _close_mutagen_audio(audio) @@ -542,8 +600,31 @@ def execute_consolidation_plan( playlists_dir = config.playlists_dir or os.path.join(config.library_root, "Playlists") quarantine_dir = config.quarantine_dir or os.path.join(config.library_root, "Quarantine") - def _record(step: str, target: str, status: str, detail: str, **metadata: object) -> None: - actions.append(ExecutionAction(step=step, target=target, status=status, detail=detail, metadata=metadata)) + def _record( + step: str, + target: str, + status: str, + detail: str, + *, + planned: bool = False, + attempted: bool = False, + verified: bool = False, + verification_detail: str | None = None, + **metadata: object, + ) -> None: + actions.append( + ExecutionAction( + step=step, + target=target, + status=status, + detail=detail, + planned=planned, + attempted=attempted, + verified=verified, + verification_detail=verification_detail, + metadata=metadata, + ) + ) def _check_cancel(stage: str) -> None: if cancel.is_set(): @@ -763,30 +844,57 @@ def _find_loser_references( playlist_backups[playlist] = working_copy playlist_targets[playlist] = working_copy playlist_base_dirs[playlist] = os.path.dirname(playlist) + verified = os.path.exists(working_copy) + verification_detail = ( + "Playlist copy verified." if verified else "Playlist copy missing after write." + ) _record( "backup_playlists", playlist, - "success", + "success" if verified else "failed", "Playlist copied for dry-run validation.", + planned=True, + attempted=True, + verified=verified, + verification_detail=verification_detail, backup_path=working_copy, group_ids=sorted(g for g in playlist_groups.get(playlist, []) if g), ) + if not verified: + raise RuntimeError(f"Playlist copy verification failed for {playlist}") else: backup_path = _backup_playlist(playlist, backups_dir) playlist_backups[playlist] = backup_path playlist_targets[playlist] = playlist playlist_base_dirs[playlist] = os.path.dirname(playlist) + verified = os.path.exists(backup_path) + verification_detail = ( + "Playlist backup verified." if verified else "Playlist backup missing after write." + ) _record( "backup_playlists", playlist, - "success", + "success" if verified else "failed", "Playlist backed up.", + planned=True, + attempted=True, + verified=verified, + verification_detail=verification_detail, backup_path=backup_path, group_ids=sorted(g for g in playlist_groups.get(playlist, []) if g), ) + if not verified: + raise RuntimeError(f"Playlist backup verification failed for {playlist}") except Exception as exc: success = False - _record("backup_playlists", playlist, "failed", f"Failed to backup playlist: {exc}") + _record( + "backup_playlists", + playlist, + "failed", + f"Failed to backup playlist: {exc}", + planned=True, + attempted=True, + ) raise # Step 2: rewrite playlists @@ -811,12 +919,20 @@ def _find_loser_references( target_playlist, "skipped", "No entries required updates.", + planned=True, + attempted=False, + verified=False, + verification_detail="No rewrite needed.", group_ids=sorted(g for g in playlist_groups.get(playlist, []) if g), original_playlist=playlist if target_playlist != playlist else None, ) continue try: _write_playlist_atomic(target_playlist, new_lines) + verified, verification_detail = _verify_playlist_write(target_playlist, new_lines) + status = "success" if verified else "failed" + if not verified: + success = False playlist_results.append( PlaylistRewriteResult( playlist=target_playlist, @@ -828,22 +944,35 @@ def _find_loser_references( ), ), replaced_entries=replaced, - status="success", + status=status, original_playlist=playlist if target_playlist != playlist else None, ) ) _record( "rewrite_playlists", target_playlist, - "success", + status, "Playlist rewritten.", + planned=True, + attempted=True, + verified=verified, + verification_detail=verification_detail, replaced=replaced, group_ids=sorted(g for g in playlist_groups.get(playlist, []) if g), original_playlist=playlist if target_playlist != playlist else None, ) + if not verified: + raise RuntimeError(f"Playlist rewrite verification failed for {target_playlist}") except Exception as exc: success = False - _record("rewrite_playlists", target_playlist, "failed", f"Failed to rewrite playlist: {exc}") + _record( + "rewrite_playlists", + target_playlist, + "failed", + f"Failed to rewrite playlist: {exc}", + planned=True, + attempted=True, + ) raise # Step 3: validate rewritten playlists @@ -862,6 +991,7 @@ def _find_loser_references( result.playlist, "failed", "Playlist validation failed; restored backup.", + attempted=True, missing=missing, group_ids=sorted(g for g in playlist_groups.get(result.original_playlist or result.playlist, []) if g), original_playlist=result.original_playlist, @@ -872,6 +1002,7 @@ def _find_loser_references( result.playlist, "success", "Playlist validated.", + attempted=True, group_ids=sorted(g for g in playlist_groups.get(result.original_playlist or result.playlist, []) if g), original_playlist=result.original_playlist, ) @@ -898,26 +1029,43 @@ def _find_loser_references( art.target, "skipped", f"Dry-run execute enabled; artwork not applied (source: {art.source}).", + planned=True, + attempted=False, + verified=False, + verification_detail="Dry-run execute enabled.", source=art.source, group_id=path_to_group.get(art.target), + skip_reason="dry_run", ) elif config.apply_artwork: for art in artwork_actions: _check_cancel("artwork") ok, detail = _apply_artwork(art) - status = "success" if ok else "failed" - if not ok: + verified = False + verification_detail = None + if ok: + verified, verification_detail = _verify_artwork_write(art) + status = "success" if ok and verified else "failed" + if status == "failed": success = False + if ok and verification_detail: + detail = f"{detail} Verification failed: {verification_detail}" _record( "artwork", art.target, status, f"{detail or art.reason} (source: {art.source})", + planned=True, + attempted=True, + verified=verified, + verification_detail=verification_detail, source=art.source, group_id=path_to_group.get(art.target), ) if not ok: raise RuntimeError(f"Artwork copy failed for {art.target}") + if ok and not verified: + raise RuntimeError(f"Artwork verification failed for {art.target}") # Step 5: apply metadata normalization if config.dry_run_execute and config.apply_metadata: @@ -927,18 +1075,69 @@ def _find_loser_references( target, "skipped", "Dry-run execute enabled; metadata not written.", + planned=True, + attempted=False, + verified=False, + verification_detail="Dry-run execute enabled.", group_id=path_to_group.get(target), + skip_reason="dry_run", ) elif config.apply_metadata: for target, tags in planned_tags.items(): _check_cancel("metadata") - ok, detail = _apply_metadata(target, tags) - status = "success" if ok else "failed" - if not ok: + if not os.path.exists(target): + _record( + "metadata", + target, + "skipped", + "Missing source; metadata not written.", + planned=True, + attempted=False, + verified=False, + verification_detail="Source missing.", + group_id=path_to_group.get(target), + skip_reason="missing_source", + ) + continue + if MutagenFile is None: + _record( + "metadata", + target, + "skipped", + "Mutagen unavailable; metadata not written.", + planned=True, + attempted=False, + verified=False, + verification_detail="Mutagen unavailable.", + group_id=path_to_group.get(target), + skip_reason="mutagen_unavailable", + ) + continue + ok, detail, skipped_keys = _apply_metadata(target, tags) + verified = False + verification_detail = None + if ok: + verified, verification_detail = _verify_metadata_write(target, tags, skipped_keys) + status = "success" if ok and verified else "failed" + if status == "failed": success = False - _record("metadata", target, status, detail, group_id=path_to_group.get(target)) + if ok and verification_detail: + detail = f"{detail} Verification failed: {verification_detail}" + _record( + "metadata", + target, + status, + detail, + planned=True, + attempted=True, + verified=verified, + verification_detail=verification_detail, + group_id=path_to_group.get(target), + ) if not ok: raise RuntimeError(f"Metadata normalization failed for {target}") + if ok and not verified: + raise RuntimeError(f"Metadata verification failed for {target}") # Step 6: quarantine or delete losers for loser, disposition in loser_disposition.items(): @@ -956,8 +1155,13 @@ def _find_loser_references( loser, "skipped", detail, + planned=True, + attempted=False, + verified=False, + verification_detail="Loser retained.", disposition=effective, group_id=path_to_group.get(loser), + skip_reason="retain", ) continue if config.dry_run_execute: @@ -967,23 +1171,53 @@ def _find_loser_references( loser, "skipped", "Dry-run execute enabled; loser not moved or deleted.", + planned=True, + attempted=False, + verified=False, + verification_detail="Dry-run execute enabled.", + disposition=effective, + group_id=path_to_group.get(loser), + skip_reason="dry_run", + ) + continue + if not os.path.exists(loser): + _record( + "loser_cleanup", + loser, + "skipped", + "Missing source; loser not moved or deleted.", + planned=True, + attempted=False, + verified=False, + verification_detail="Source missing.", disposition=effective, group_id=path_to_group.get(loser), + skip_reason="missing_source", ) continue if effective == "delete": try: - if os.path.exists(loser): - os.remove(loser) - quarantine_index[loser] = "deleted" + os.remove(loser) + verified = not os.path.exists(loser) + verification_detail = ( + "Loser deletion verified." if verified else "Loser still present after delete." + ) + if verified: + quarantine_index[loser] = "deleted" _record( "loser_cleanup", loser, - "success", + "success" if verified else "failed", "Loser deleted.", + planned=True, + attempted=True, + verified=verified, + verification_detail=verification_detail, disposition=effective, group_id=path_to_group.get(loser), ) + if not verified: + raise RuntimeError(f"Loser delete verification failed for {loser}") except Exception as exc: success = False _record( @@ -991,6 +1225,8 @@ def _find_loser_references( loser, "failed", f"Failed to delete loser: {exc}", + planned=True, + attempted=True, disposition=effective, group_id=path_to_group.get(loser), ) @@ -1003,18 +1239,28 @@ def _find_loser_references( flatten=config.quarantine_flatten, ) try: - if os.path.exists(loser): - shutil.move(loser, dest) - quarantine_index[loser] = dest + shutil.move(loser, dest) + verified = os.path.exists(dest) and not os.path.exists(loser) + verification_detail = ( + "Loser quarantine verified." if verified else "Quarantine move not confirmed." + ) + if verified: + quarantine_index[loser] = dest _record( "loser_cleanup", loser, - "success", + "success" if verified else "failed", "Loser quarantined.", + planned=True, + attempted=True, + verified=verified, + verification_detail=verification_detail, quarantine_path=dest, disposition=effective, group_id=path_to_group.get(loser), ) + if not verified: + raise RuntimeError(f"Loser quarantine verification failed for {loser}") except Exception as exc: success = False _record( @@ -1022,6 +1268,8 @@ def _find_loser_references( loser, "failed", f"Failed to quarantine loser: {exc}", + planned=True, + attempted=True, quarantine_path=dest, disposition=effective, group_id=path_to_group.get(loser), @@ -1039,6 +1287,13 @@ def _find_loser_references( plan_signature = plan.plan_signature if plan else None source_snapshot = plan.source_snapshot if plan else {} + rollup = { + "planned": sum(1 for a in actions if a.planned), + "attempted": sum(1 for a in actions if a.attempted), + "verified": sum(1 for a in actions if a.verified), + "skipped": sum(1 for a in actions if a.status == "skipped"), + "failed": sum(1 for a in actions if a.status in {"failed", "blocked"}), + } consolidated_payload = { "success": success, "actions": [a.to_dict() for a in actions], @@ -1050,6 +1305,12 @@ def _find_loser_references( "playlists": [p.to_dict() for p in playlist_results], "quarantine_index": dict(quarantine_index), "metadata_plans": {path: dict(tags) for path, tags in planned_tags.items()}, + "action_rollup": dict(rollup), + "execution_context": { + "dry_run_execute": config.dry_run_execute, + "library_root": os.path.abspath(config.library_root), + "quarantine_dir": os.path.abspath(quarantine_dir) if quarantine_dir else None, + }, "run_paths": { "reports_dir": reports_dir, "backups_dir": backups_dir, @@ -1395,6 +1656,14 @@ def _trace_panel(group: object) -> List[str]: winners_kept = groups_processed generated_at_iso = datetime.datetime.now(datetime.timezone.utc).isoformat() host_name = platform.node() or "unknown-host" + planned_count = rollup.get("planned", 0) + attempted_count = rollup.get("attempted", 0) + verified_count = rollup.get("verified", 0) + skipped_count = rollup.get("skipped", 0) + failed_count = rollup.get("failed", 0) + execution_mode = "dry-run" if config.dry_run_execute else "real" + resolved_library_root = os.path.abspath(config.library_root) + resolved_quarantine_dir = os.path.abspath(quarantine_dir) if quarantine_dir else "n/a" def _status_badge_class(status: str) -> str: if status == "success": @@ -1994,6 +2263,13 @@ def _is_artwork_variant(group: object, loser_path: str, threshold: float) -> boo "
      ", "
      ", "
      ", + f"Planned {planned_count}", + f"Attempted {attempted_count}", + f"Verified {verified_count}", + f"Skipped {skipped_count}", + f"Failed {failed_count}", + "
      ", + "
      ", f"Exact {total_exact}", f"Near {total_near}", "
      ", @@ -2003,6 +2279,12 @@ def _is_artwork_variant(group: object, loser_path: str, threshold: float) -> boo f"
      {html.escape(plan_signature or computed_signature)}
      ", "
      Reports directory
      ", f"
      {html.escape(reports_dir)}
      ", + "
      Execution mode
      ", + f"
      {html.escape(execution_mode)}
      ", + "
      Library root
      ", + f"
      {html.escape(resolved_library_root)}
      ", + "
      Quarantine dir
      ", + f"
      {html.escape(resolved_quarantine_dir)}
      ", "
      ", "
      ", "
      ", diff --git a/main_gui.py b/main_gui.py index 59859e5..93f5721 100644 --- a/main_gui.py +++ b/main_gui.py @@ -2975,9 +2975,16 @@ def finish(result, error: Exception | None = None) -> None: self._set_status("Execution failed", progress=100) messagebox.showerror("Execution Failed", str(error)) return - status = "Executed" if result.success else "Execution failed" + cancelled = any(action.status == "cancelled" for action in result.actions) + if cancelled: + status = "Execution cancelled" + else: + status = "Executed" if result.success else "Execution failed" self._set_status(status, progress=100) - self._log_action(f"Execution complete: {'success' if result.success else 'failed'}") + if cancelled: + self._log_action("Execution complete: cancelled") + else: + self._log_action(f"Execution complete: {'success' if result.success else 'failed'}") report_path = result.report_paths.get("html_report") self._log_action(f"Execution report: {report_path}") if report_path: @@ -2988,7 +2995,22 @@ def finish(result, error: Exception | None = None) -> None: else: self.execution_report_path = None self.open_report_btn.config(state="disabled") - if not result.success: + if cancelled: + report_line = "" + if report_path: + if os.path.exists(ensure_long_path(report_path)): + report_line = f"\n\nReport (HTML): {report_path}" + else: + report_line = ( + "\n\nReport (HTML) was not found at the expected path:\n" + f"{report_path}" + ) + messagebox.showinfo( + "Execution Cancelled", + "Execution was cancelled. Review the report for details." + f"{report_line}", + ) + elif not result.success: report_line = "" if report_path: if os.path.exists(ensure_long_path(report_path)): From 3a2b547906a98a4a5e6774e01feade30cfac4091 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 5 Jan 2026 21:06:35 -0500 Subject: [PATCH 292/606] Add execute confirmation step to duplicate finder --- main_gui.py | 48 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/main_gui.py b/main_gui.py index 93f5721..9d1ebcf 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1969,7 +1969,8 @@ def __init__(self, parent: tk.Widget, library_path: str): self.update_playlists_var = tk.BooleanVar(value=False) self.quarantine_var = tk.BooleanVar(value=True) self.delete_losers_var = tk.BooleanVar(value=False) - self.override_review_var = tk.BooleanVar(value=False) + self._execute_confirmation_pending = False + self.execute_label_var = tk.StringVar(value="Execute") cfg = load_config() self.show_artwork_variants_var = tk.BooleanVar( value=cfg.get("duplicate_finder_show_artwork_variants", True) @@ -2042,7 +2043,12 @@ def __init__(self, parent: tk.Widget, library_path: str): controls, text="Open Report", command=self._open_execution_report, state="disabled" ) self.open_report_btn.pack(side="left", padx=(0, 6)) - ttk.Button(controls, text="Execute", command=self._handle_execute).pack( + self.execute_btn = ttk.Button( + controls, + textvariable=self.execute_label_var, + command=self._handle_execute, + ) + self.execute_btn.pack( side="left", padx=(0, 12) ) ttk.Button(controls, text="⚙️ Thresholds", command=self._open_threshold_settings).pack( @@ -2072,11 +2078,6 @@ def __init__(self, parent: tk.Widget, library_path: str): variable=self.delete_losers_var, command=self._toggle_delete_losers, ).pack(side="left", padx=(10, 0)) - ttk.Checkbutton( - controls, - text="Override review blocks", - variable=self.override_review_var, - ).pack(side="left", padx=(10, 0)) # Progress + status status_frame = ttk.Frame(container) @@ -2104,6 +2105,8 @@ def __init__(self, parent: tk.Widget, library_path: str): groups_frame = ttk.LabelFrame(results, text="Duplicate Groups") groups_frame.grid(row=0, column=0, sticky="nsew", padx=(6, 3), pady=6) + groups_frame.configure(height=320) + groups_frame.grid_propagate(False) ttk.Label( groups_frame, textvariable=self.fingerprint_status_var, @@ -2155,9 +2158,14 @@ def _clear_plan(self, note: str | None = None) -> None: self.group_details.configure(state="disabled") self._reset_group_selection() self._set_fingerprint_status("Ready.") + self._reset_execute_confirmation() if note: self._log_action(note) + def _reset_execute_confirmation(self) -> None: + self._execute_confirmation_pending = False + self.execute_label_var.set("Execute") + def _default_playlist_folder(self, library_path: str) -> str: if library_path: candidate = os.path.join(library_path, "Playlists") @@ -2364,6 +2372,7 @@ def _on_group_disposition_change(self, event=None) -> None: self._plan.refresh_plan_signature() self._render_group_details(group) self._reset_preview_if_needed() + self._reset_execute_confirmation() def _count_group_dispositions(self) -> dict[str, int]: counts = {"retain": 0, "quarantine": 0, "delete": 0} @@ -2629,6 +2638,7 @@ def _save() -> None: self.preview_html_path = None self.execution_report_path = None self.open_report_btn.config(state="disabled") + self._reset_execute_confirmation() library_root = self.library_path_var.get().strip() if library_root: docs_dir = os.path.join(library_root, "Docs") @@ -2704,8 +2714,7 @@ def _gather_tracks(self, library_root: str) -> list[dict[str, object]]: pass now = time.monotonic() if idx == total or now - last_update >= update_interval: - label = self._format_track_label(path, audio) - self._set_fingerprint_status(f"Fingerprinting {idx}/{total} — {label}") + self._set_fingerprint_status(f"Fingerprinting {idx}/{total}") self._set_status( "Fingerprinting…", progress=self._weighted_progress("fingerprinting", idx / total), @@ -2815,16 +2824,23 @@ def _handle_scan(self) -> None: path = self._validate_library_root() if not path: return + self._reset_execute_confirmation() self._log_action("Scan clicked") self._generate_plan(write_preview=False) def _handle_preview(self) -> None: + self._reset_execute_confirmation() self._log_action("Preview clicked") self._generate_plan(write_preview=True) if self.preview_html_path: self._open_preview() def _handle_execute(self) -> None: + if not self._execute_confirmation_pending: + self._execute_confirmation_pending = True + self.execute_label_var.set("Confirm Execute") + self._log_action("Execute confirmation requested.") + return path = self._validate_library_root() if not path: return @@ -2904,10 +2920,11 @@ def _handle_execute(self) -> None: messagebox.showwarning("Preview Required", "Generate a preview before executing.") self._log_action("Execute blocked: no plan generated") return - if plan_for_checks and plan_for_checks.review_required_count and not self.override_review_var.get(): + allow_review_required = self._execute_confirmation_pending + if plan_for_checks and plan_for_checks.review_required_count and not allow_review_required: messagebox.showwarning( "Review Required", - "Resolve review-required groups or check Override review blocks to proceed.", + "Resolve review-required groups or confirm execution again to bypass the review block.", ) self._log_action("Execute blocked: review required groups pending") return @@ -2949,7 +2966,7 @@ def log_callback(msg: str) -> None: quarantine_dir=os.path.join(path, "Quarantine"), quarantine_flatten=True, log_callback=log_callback, - allow_review_required=self.override_review_var.get(), + allow_review_required=allow_review_required, retain_losers=not self.quarantine_var.get(), allow_deletion=deletions_requested, confirm_deletion=deletions_requested, @@ -2968,12 +2985,14 @@ def log_callback(msg: str) -> None: self._set_status("Executing…", progress=10) self._log_action("Execute started") + self._reset_execute_confirmation() def finish(result, error: Exception | None = None) -> None: if error is not None: self._log_action(f"Execution failed: {error}") self._set_status("Execution failed", progress=100) messagebox.showerror("Execution Failed", str(error)) + self._reset_execute_confirmation() return cancelled = any(action.status == "cancelled" for action in result.actions) if cancelled: @@ -3025,6 +3044,7 @@ def finish(result, error: Exception | None = None) -> None: "Execution completed with errors. Review the report for details." f"{report_line}", ) + self._reset_execute_confirmation() def worker() -> None: try: @@ -3039,6 +3059,7 @@ def worker() -> None: def _toggle_update_playlists(self) -> None: state = "enabled" if self.update_playlists_var.get() else "disabled" self._log_action(f"Update Playlists {state}") + self._reset_execute_confirmation() def _toggle_show_artwork_variants(self) -> None: enabled = self.show_artwork_variants_var.get() @@ -3047,6 +3068,7 @@ def _toggle_show_artwork_variants(self) -> None: save_config(cfg) state = "enabled" if enabled else "disabled" self._log_action(f"Show different artwork variants {state}") + self._reset_execute_confirmation() def _toggle_quarantine(self) -> None: state = "enabled" if self.quarantine_var.get() else "disabled" @@ -3059,6 +3081,7 @@ def _toggle_quarantine(self) -> None: self.quarantine_var.set(True) self._log_action("Quarantine Duplicates enabled to support deletions.") self._reset_preview_if_needed() + self._reset_execute_confirmation() def _toggle_delete_losers(self) -> None: if self.delete_losers_var.get(): @@ -3070,6 +3093,7 @@ def _toggle_delete_losers(self) -> None: self._log_action("Delete losers disabled; preview will quarantine losers.") self._apply_deletion_mode() self._reset_preview_if_needed() + self._reset_execute_confirmation() def _apply_deletion_mode(self) -> None: if not self._plan: From 0bd34432260d302299b0722276bede2517b857f7 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 5 Jan 2026 21:31:26 -0500 Subject: [PATCH 293/606] Respect playlist update setting during cleanup --- duplicate_consolidation_executor.py | 222 +++++++++++++++++++++++++++- 1 file changed, 221 insertions(+), 1 deletion(-) diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index c2dc2ae..2ed9a52 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -21,6 +21,7 @@ import logging import platform import os +import re import shutil import tempfile import threading @@ -148,6 +149,7 @@ class ExecutionResult: EXECUTION_REPORT_FILENAME = "execution_report.html" INTERNAL_TAG_KEYS = {"album_type"} +_NUMERIC_SUFFIX_RE = re.compile(r"\s*\((\d+)\)$") def _timestamped_dir(base: str) -> str: @@ -281,6 +283,18 @@ def _backup_playlist(path: str, backup_root: str) -> str: return dest +def _strip_trailing_numeric_suffix(path: str) -> str | None: + base = os.path.basename(path) + stem, ext = os.path.splitext(base) + match = _NUMERIC_SUFFIX_RE.search(stem) + if not match: + return None + trimmed = stem[: match.start()].rstrip() + if not trimmed: + return None + return os.path.join(os.path.dirname(path), f"{trimmed}{ext}") + + def _extract_artwork_bytes(path: str) -> bytes | None: payload = read_sidecar_artwork_bytes(path) if payload is not None: @@ -597,7 +611,12 @@ def execute_consolidation_plan( os.makedirs(reports_dir, exist_ok=True) os.makedirs(backups_dir, exist_ok=True) - playlists_dir = config.playlists_dir or os.path.join(config.library_root, "Playlists") + playlists_dir = ( + config.playlists_dir + if config.playlists_dir is not None + else os.path.join(config.library_root, "Playlists") + ) + update_playlists = bool(playlists_dir) quarantine_dir = config.quarantine_dir or os.path.join(config.library_root, "Quarantine") def _record( @@ -1275,6 +1294,207 @@ def _find_loser_references( group_id=path_to_group.get(loser), ) raise + + # Step 7: cleanup redundant numeric suffixes on winners + cleanup_candidates: List[tuple[str, str]] = [] + for group in plan.groups if plan else []: + candidate = _strip_trailing_numeric_suffix(group.winner_path) + if candidate and candidate != group.winner_path: + cleanup_candidates.append((group.winner_path, candidate)) + + rename_map: Dict[str, str] = {} + if cleanup_candidates: + if config.dry_run_execute: + for original, candidate in cleanup_candidates: + _record( + "filename_cleanup", + original, + "skipped", + "Dry-run execute enabled; filename cleanup not applied.", + planned=True, + attempted=False, + verified=False, + verification_detail="Dry-run execute enabled.", + proposed_path=candidate, + group_id=path_to_group.get(original), + skip_reason="dry_run", + ) + else: + for original, candidate in cleanup_candidates: + _check_cancel("filename_cleanup") + if not os.path.exists(original): + _record( + "filename_cleanup", + original, + "skipped", + "Missing source; filename cleanup not applied.", + planned=True, + attempted=False, + verified=False, + verification_detail="Source missing.", + proposed_path=candidate, + group_id=path_to_group.get(original), + skip_reason="missing_source", + ) + continue + if os.path.exists(candidate): + _record( + "filename_cleanup", + original, + "skipped", + "Filename cleanup skipped to avoid collision.", + planned=True, + attempted=False, + verified=False, + verification_detail="Destination already exists.", + proposed_path=candidate, + group_id=path_to_group.get(original), + skip_reason="collision", + ) + continue + try: + os.replace( + ensure_long_path(original), + ensure_long_path(candidate), + ) + verified = os.path.exists(candidate) and not os.path.exists(original) + verification_detail = ( + "Filename cleanup verified." + if verified + else "Cleanup rename not confirmed." + ) + status = "success" if verified else "failed" + if verified: + rename_map[original] = candidate + else: + success = False + _record( + "filename_cleanup", + original, + status, + "Removed redundant numeric suffix from filename.", + planned=True, + attempted=True, + verified=verified, + verification_detail=verification_detail, + new_path=candidate, + group_id=path_to_group.get(original), + ) + if not verified: + raise RuntimeError(f"Filename cleanup verification failed for {original}") + except Exception as exc: + success = False + _record( + "filename_cleanup", + original, + "failed", + f"Failed to cleanup filename: {exc}", + planned=True, + attempted=True, + proposed_path=candidate, + group_id=path_to_group.get(original), + ) + raise + + if rename_map: + for old_path, new_path in rename_map.items(): + group_id = path_to_group.pop(old_path, None) + if group_id: + path_to_group[new_path] = group_id + if not update_playlists: + _record( + "filename_cleanup_playlists", + "playlists", + "skipped", + "Playlist updates disabled; filename cleanup not applied to playlists.", + planned=True, + attempted=False, + verified=False, + verification_detail="Playlist updates disabled.", + skip_reason="playlists_disabled", + ) + else: + for playlist in _iter_playlists(playlists_dir): + _check_cancel("filename_cleanup_playlists") + base_dir = os.path.dirname(playlist) + replaced, new_lines = _rewrite_playlist(playlist, rename_map, base_dir=base_dir) + if replaced == 0: + continue + if playlist not in playlist_backups: + try: + backup_path = _backup_playlist(playlist, backups_dir) + playlist_backups[playlist] = backup_path + _record( + "backup_playlists", + playlist, + "success", + "Playlist backed up for filename cleanup.", + planned=True, + attempted=True, + verified=os.path.exists(backup_path), + verification_detail="Backup created for filename cleanup.", + backup_path=backup_path, + ) + except Exception as exc: + success = False + _record( + "backup_playlists", + playlist, + "failed", + f"Failed to backup playlist for cleanup: {exc}", + planned=True, + attempted=True, + ) + raise + try: + _write_playlist_atomic(playlist, new_lines) + verified, verification_detail = _verify_playlist_write(playlist, new_lines) + status = "success" if verified else "failed" + if not verified: + success = False + _record( + "filename_cleanup_playlists", + playlist, + status, + "Playlist rewritten for filename cleanup.", + planned=True, + attempted=True, + verified=verified, + verification_detail=verification_detail, + replaced=replaced, + ) + if not verified: + raise RuntimeError(f"Playlist cleanup rewrite verification failed for {playlist}") + missing = _validate_playlist(playlist, base_dir=base_dir) + if missing: + success = False + _record( + "filename_cleanup_validate", + playlist, + "failed", + "Playlist validation failed after filename cleanup.", + attempted=True, + missing=missing, + ) + raise RuntimeError(f"Playlist cleanup validation failed for {playlist}") + _record( + "filename_cleanup_validate", + playlist, + "success", + "Playlist validated after filename cleanup.", + attempted=True, + ) + except Exception as exc: + success = False + _record( + "filename_cleanup_playlists", + playlist, + "failed", + f"Failed to rewrite playlist for cleanup: {exc}", + planned=True, + attempted=True, + ) + raise except IndexCancelled: success = False _record("execution", "execution", "cancelled", "Execution cancelled by user request.") From e7b158ca0d657eaf61f145acc7790b59e5d5d33f Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Mon, 5 Jan 2026 22:30:45 -0500 Subject: [PATCH 294/606] Serialize duplicate finder fingerprint cache access --- fingerprint_cache.py | 108 +++++++++++++++++++++++++++++++++++++ main_gui.py | 123 ++++++++++++++++++++++++++++++++----------- 2 files changed, 201 insertions(+), 30 deletions(-) diff --git a/fingerprint_cache.py b/fingerprint_cache.py index 2ce3b4f..1bbb348 100644 --- a/fingerprint_cache.py +++ b/fingerprint_cache.py @@ -120,6 +120,114 @@ def get_fingerprint( return fp_hash +def get_cached_fingerprint( + path: str, + db_path: str, + log_callback: Optional[Callable[[str], None]] = None, + trace: Optional[Dict[str, object]] = None, + *, + retries: int = 3, + retry_delay: float = 0.05, +) -> Optional[str]: + """Return cached fingerprint for path without computing new fingerprints.""" + if log_callback is None: + log_callback = lambda msg: None + if trace is None: + trace = {} + + path = ensure_long_path(path) + os.makedirs(os.path.dirname(db_path), exist_ok=True) + for attempt in range(retries): + conn: sqlite3.Connection | None = None + try: + conn = _ensure_db(db_path) + try: + mtime = os.path.getmtime(path) + size = os.path.getsize(path) + except OSError as e: + log_callback(f"! Could not stat {path}: {e}") + trace["source"] = "stat_error" + trace["error"] = str(e) + return None + row = conn.execute( + "SELECT mtime, size, fingerprint FROM fingerprints WHERE path=?", + (path,), + ).fetchone() + cached_mtime = row[0] if row else None + cached_size = row[1] if row else None + if row and abs(cached_mtime - mtime) < 1e-6 and int(cached_size or 0) == int(size): + fp = row[2] + _dlog("FP", f"cache hit {path}", log_callback) + if isinstance(fp, (bytes, bytearray)): + try: + fp = fp.decode("utf-8") + except Exception: + fp = fp.decode("latin1", errors="ignore") + _dlog("FP", f"fingerprint={fp} prefix={fp[:16]}", log_callback) + trace["source"] = "cache" + trace["error"] = "" + return fp + trace["source"] = "missing" + trace["error"] = "" + return None + except sqlite3.OperationalError as e: + if "locked" not in str(e).lower() or attempt >= retries - 1: + log_callback(f"! Fingerprint cache read failed: {e}") + trace["source"] = "cache_error" + trace["error"] = str(e) + return None + time.sleep(retry_delay) + finally: + if conn is not None: + conn.close() + return None + + +def store_fingerprint( + path: str, + db_path: str, + duration: int | None, + fingerprint: str | None, + log_callback: Optional[Callable[[str], None]] = None, + *, + retries: int = 3, + retry_delay: float = 0.05, +) -> bool: + """Persist a fingerprint in the cache without computing it.""" + if fingerprint is None: + return False + if log_callback is None: + log_callback = lambda msg: None + + path = ensure_long_path(path) + os.makedirs(os.path.dirname(db_path), exist_ok=True) + for attempt in range(retries): + conn: sqlite3.Connection | None = None + try: + conn = _ensure_db(db_path) + try: + mtime = os.path.getmtime(path) + size = os.path.getsize(path) + except OSError as e: + log_callback(f"! Could not stat {path}: {e}") + return False + conn.execute( + "INSERT OR REPLACE INTO fingerprints (path, mtime, size, duration, fingerprint) VALUES (?, ?, ?, ?, ?)", + (path, mtime, size, duration, fingerprint), + ) + conn.commit() + return True + except sqlite3.OperationalError as e: + if "locked" not in str(e).lower() or attempt >= retries - 1: + log_callback(f"! Fingerprint cache write failed: {e}") + return False + time.sleep(retry_delay) + finally: + if conn is not None: + conn.close() + return False + + def flush_cache(db_path: str) -> None: if not os.path.exists(db_path): return diff --git a/main_gui.py b/main_gui.py index 9d1ebcf..444673c 100644 --- a/main_gui.py +++ b/main_gui.py @@ -37,6 +37,7 @@ import textwrap import time from datetime import datetime +from concurrent.futures import ThreadPoolExecutor, as_completed from validator import validate_soundvault_structure from music_indexer_api import ( @@ -78,7 +79,7 @@ NEAR_DUPLICATE_THRESHOLD, load_config, ) -from fingerprint_cache import get_fingerprint +from fingerprint_cache import get_fingerprint, get_cached_fingerprint, store_fingerprint from simple_duplicate_finder import SUPPORTED_EXTS, _compute_fp from tag_fixer import MIN_INTERACTIVE_SCORE, FileRecord from typing import Any, Callable, List, Mapping @@ -2688,17 +2689,18 @@ def _gather_tracks(self, library_root: str) -> list[dict[str, object]]: self.update_idletasks() last_update = time.monotonic() update_interval = 0.2 - for idx, path in enumerate(sorted(audio_paths), start=1): - fingerprint_trace: dict[str, object] = {} - fp = get_fingerprint( - path, - db_path, - _compute_fp, - log_callback=self._log_action, - trace=fingerprint_trace, - ) - if not fp: - continue + def _fingerprint_worker(target_path: str) -> tuple[str, int | None, str | None, str | None]: + try: + duration, fp = _compute_fp(target_path) + return target_path, duration, fp, None + except Exception as exc: # pragma: no cover - depends on external files + return target_path, None, None, str(exc) + + def _track_payload( + path: str, + fp: str, + fingerprint_trace: dict[str, object], + ) -> dict[str, object]: bitrate = 0 sample_rate = 0 bit_depth = 0 @@ -2712,31 +2714,92 @@ def _gather_tracks(self, library_root: str) -> list[dict[str, object]]: bit_depth = int(getattr(info, "bits_per_sample", 0) or getattr(info, "bitdepth", 0) or 0) except Exception: pass + return { + "path": path, + "fingerprint": fp, + "ext": os.path.splitext(path)[1].lower(), + "bitrate": bitrate, + "sample_rate": sample_rate, + "bit_depth": bit_depth, + "fingerprint_trace": dict(fingerprint_trace), + "discovery": { + "scan_roots": [library_root], + "excluded_dirs": sorted(excluded_dirs), + "skipped_by_filter": False, + }, + } + + tracks_map: dict[str, dict[str, object]] = {} + pending_paths: list[str] = [] + completed = 0 + failure_count = 0 + missing_count = 0 + sorted_paths = sorted(audio_paths) + for path in sorted_paths: + fingerprint_trace: dict[str, object] = {} + fp = get_cached_fingerprint(path, db_path, log_callback=self._log_action, trace=fingerprint_trace) + if fp: + tracks_map[path] = _track_payload(path, fp, fingerprint_trace) + completed += 1 + else: + source = fingerprint_trace.get("source") + if source in {"stat_error", "cache_error"}: + failure_count += 1 + completed += 1 + else: + pending_paths.append(path) now = time.monotonic() - if idx == total or now - last_update >= update_interval: - self._set_fingerprint_status(f"Fingerprinting {idx}/{total}") + if completed == total or now - last_update >= update_interval: + self._set_fingerprint_status(f"Fingerprinting {completed}/{total}") self._set_status( "Fingerprinting…", - progress=self._weighted_progress("fingerprinting", idx / total), + progress=self._weighted_progress("fingerprinting", completed / total), ) self.update_idletasks() last_update = now - tracks.append( - { - "path": path, - "fingerprint": fp, - "ext": os.path.splitext(path)[1].lower(), - "bitrate": bitrate, - "sample_rate": sample_rate, - "bit_depth": bit_depth, - "fingerprint_trace": dict(fingerprint_trace), - "discovery": { - "scan_roots": [library_root], - "excluded_dirs": sorted(excluded_dirs), - "skipped_by_filter": False, - }, - } + + max_workers = min(8, os.cpu_count() or 4) + if pending_paths: + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = {executor.submit(_fingerprint_worker, path): path for path in pending_paths} + for fut in as_completed(futures): + path = futures[fut] + error = None + try: + path, duration, fp, error = fut.result() + except Exception as exc: + duration = None + fp = None + error = str(exc) + completed += 1 + now = time.monotonic() + if completed == total or now - last_update >= update_interval: + self._set_fingerprint_status(f"Fingerprinting {completed}/{total}") + self._set_status( + "Fingerprinting…", + progress=self._weighted_progress("fingerprinting", completed / total), + ) + self.update_idletasks() + last_update = now + if error: + failure_count += 1 + self._log_action(f"! Fingerprint failed for {path}: {error}") + continue + if not fp: + missing_count += 1 + self._log_action(f"! Missing fingerprint for {path}") + continue + if not store_fingerprint(path, db_path, duration, fp, log_callback=self._log_action): + failure_count += 1 + fingerprint_trace = {"source": "computed", "error": ""} + tracks_map[path] = _track_payload(path, fp, fingerprint_trace) + + if missing_count or failure_count: + self._log_action( + f"Fingerprinting complete with {missing_count} missing fingerprints and " + f"{failure_count} failures." ) + tracks = [tracks_map[path] for path in sorted_paths if path in tracks_map] return tracks def _generate_plan(self, write_preview: bool) -> None: From 3e73048bd6cbe14dcccafd156454bf17686ac0f1 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 6 Jan 2026 18:48:24 -0500 Subject: [PATCH 295/606] Refine duplicate consolidation preview actions --- duplicate_consolidation.py | 92 ++++++++++++++++++- duplicate_consolidation_executor.py | 15 ++- tests/test_duplicate_consolidation_preview.py | 2 +- 3 files changed, 99 insertions(+), 10 deletions(-) diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index a3dd9d7..f33bda0 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -1095,6 +1095,62 @@ def _expect_str_mapping(value: object, field: str) -> Dict[str, object]: return {str(k): v for k, v in mapping.items()} +def _iter_playlists(playlists_dir: str) -> Iterable[str]: + if not os.path.isdir(playlists_dir): + return [] + for dirpath, _dirs, files in os.walk(playlists_dir): + for fname in files: + if fname.lower().endswith(".m3u"): + yield os.path.join(dirpath, fname) + + +def _normalize_playlist_entry(entry: str, playlist_dir: str) -> str: + if not entry or entry.startswith("#"): + return entry + if os.path.isabs(entry): + return os.path.normpath(entry) + return os.path.normpath(os.path.join(playlist_dir, entry)) + + +def _playlist_rewrite_losers(playlists_dir: str | None, losers: Iterable[str]) -> tuple[set[str], int]: + if not playlists_dir or not os.path.isdir(playlists_dir): + return set(), 0 + loser_set = {os.path.normpath(path) for path in losers if path} + if not loser_set: + return set(), 0 + matches: set[str] = set() + playlists_with_hits = 0 + for playlist in _iter_playlists(playlists_dir): + try: + with open(playlist, "r", encoding="utf-8") as handle: + lines = [ln.rstrip("\n") for ln in handle] + except OSError: + continue + playlist_dir = os.path.dirname(playlist) + normalized_lines = {_normalize_playlist_entry(ln, playlist_dir) for ln in lines if ln} + hit = loser_set & normalized_lines + if hit: + playlists_with_hits += 1 + matches.update(hit) + if len(matches) == len(loser_set): + return matches, playlists_with_hits + return matches, playlists_with_hits + + +def _infer_playlists_dir(paths: Sequence[str]) -> str | None: + if not paths: + return None + try: + common = os.path.commonpath(paths) + except ValueError: + return None + candidate = common if os.path.isdir(common) else os.path.dirname(common) + playlists_dir = os.path.join(candidate, "Playlists") + if os.path.isdir(playlists_dir): + return playlists_dir + return None + + def build_duplicate_pair_report( track_a: Mapping[str, object], track_b: Mapping[str, object], @@ -2099,6 +2155,7 @@ def build_consolidation_plan( plans: List[GroupPlan] = [] plan_placeholders = False + playlists_dir = _infer_playlists_dir([t.path for t in normalized]) for cluster in sorted(clusters, key=lambda c: _stable_group_id([t.path for t in c.tracks])): cluster_tracks = cluster.tracks artwork_hashes = {t.path: _artwork_hash_for_track(t) for t in cluster_tracks} @@ -2410,7 +2467,8 @@ def build_consolidation_plan( ) dispositions = {loser: "quarantine" for loser in losers} - playlist_map = {loser: winner.path for loser in losers} + playlist_hits, playlist_count = _playlist_rewrite_losers(playlists_dir, losers) + playlist_map = {loser: winner.path for loser in losers if loser in playlist_hits} group_id = _stable_group_id([t.path for t in group_tracks]) if ambiguous_art and not suppress_review_flags: @@ -2434,7 +2492,7 @@ def build_consolidation_plan( group_review = [] placeholders = False - playlist_impact = PlaylistImpact(playlists=len(losers), entries=len(losers)) + playlist_impact = PlaylistImpact(playlists=playlist_count, entries=len(playlist_hits)) track_states = {t.path: dict(t.library_state) for t in group_tracks} artwork_candidates: List[ArtworkCandidate] = [] @@ -2880,6 +2938,16 @@ def _planned_actions(group: GroupPlan) -> List[Dict[str, object]]: ) return actions + def _is_noop_group(actions: List[Dict[str, object]]) -> bool: + has_non_loser_action = any( + act["step"] in {"metadata", "artwork", "playlist"} for act in actions + ) + has_non_retain_loser = any( + act["step"] == "loser_cleanup" and act.get("metadata", {}).get("disposition") != "retain" + for act in actions + ) + return not has_non_loser_action and not has_non_retain_loser + def _missing_artwork_gate(group: GroupPlan) -> bool: for note in group.artwork_evidence: lowered = note.lower() @@ -3603,6 +3671,8 @@ def _format_setting(value: object) -> str: ".muted{ color: var(--muted); }", ".tiny{ font-size: 12px; }", ".hidden{ display:none !important; }", + ".toggle{ display:flex; align-items:center; gap:6px; }", + ".empty{ margin-top:14px; padding:18px; border:1px dashed var(--border); border-radius:8px; color:var(--muted); }", "", "", "", @@ -3678,6 +3748,7 @@ def _format_setting(value: object) -> str: "", "", "", + "", "
      ", "
      ", "0 visible", @@ -3690,6 +3761,7 @@ def _format_setting(value: object) -> str: ) all_actions: List[Dict[str, object]] = [] + actionable_groups = 0 for group in plan.groups: actions = _planned_actions(group) visible_losers = list(group.losers) @@ -3707,7 +3779,10 @@ def _format_setting(value: object) -> str: ] if not show_artwork_variants and not actions and not visible_losers: continue + is_noop = _is_noop_group(actions) all_actions.extend(actions) + if not is_noop: + actionable_groups += 1 state = "review" if group.review_flags else "ready" badges = _action_badges(group, actions) group_disposition_count = sum( @@ -3741,6 +3816,7 @@ def _format_setting(value: object) -> str: "data-has-failure='false' " f"data-type='{esc(group_type_hint)}' " f"data-search='{esc(search_text.lower())}' " + f"data-no-op='{str(is_noop).lower()}' " f"data-quarantine-count='{group_disposition_count}'>" ) album_art_src = _album_art_src(group, group.winner_path) @@ -4008,6 +4084,8 @@ def _format_setting(value: object) -> str: html_lines.append("
      ") html_lines.append("") + if actionable_groups == 0: + html_lines.append("
      No changes needed.
      ") html_lines.append("") html_lines.append("
      ") html_lines.append("") @@ -4081,13 +4159,15 @@ def _format_setting(value: object) -> str: "const groups = () => $$('#groups details.group');" "const searchEl = $('#search');" "const filterEl = $('#filter');" + "const showNoOpsEl = $('#showNoOps');" "const visibleCountEl = $('#visibleCount');" "const expandAllBtn = $('#expandAll');" "const collapseAllBtn = $('#collapseAll');" "const settingsBtn = $('#toggleSettings');" "const settingsPanel = $('#settingsPanel');" "function computeStats(){" - "const g = groups();" + "const showNoOps = showNoOpsEl ? showNoOpsEl.checked : false;" + "const g = groups().filter(d => showNoOps || (d.getAttribute('data-no-op') || 'false') !== 'true');" "$('#statGroups').textContent = g.length.toString();" "$('#statWinners').textContent = g.length.toString();" "let qTotal = 0;" @@ -4117,6 +4197,7 @@ def _format_setting(value: object) -> str: "function applyFilters(){" "const term = (searchEl.value || '').trim().toLowerCase();" "const mode = filterEl.value;" + "const showNoOps = showNoOpsEl ? showNoOpsEl.checked : false;" "let visible = 0;" "for (const d of groups()){" "const hay = (d.getAttribute('data-search') || '').toLowerCase();" @@ -4124,15 +4205,17 @@ def _format_setting(value: object) -> str: "const hasQuarantine = (d.getAttribute('data-has-quarantine') || 'false') === 'true';" "const hasFailure = (d.getAttribute('data-has-failure') || 'false') === 'true';" "const typeHint = (d.getAttribute('data-type') || '').toLowerCase();" + "const isNoOp = (d.getAttribute('data-no-op') || 'false') === 'true';" "let matchesFilter = true;" "if (mode === 'has-quarantine') matchesFilter = hasQuarantine;" "if (mode === 'metadata-only') matchesFilter = typeHint === 'metadata-only';" "if (mode === 'failed') matchesFilter = hasFailure;" - "const show = matchesSearch && matchesFilter;" + "const show = matchesSearch && matchesFilter && (showNoOps || !isNoOp);" "d.classList.toggle('hidden', !show);" "if (show) visible += 1;" "}" "visibleCountEl.textContent = `${visible} visible`;" + "computeStats();" "}" "for (const btn of $$('button.copy[data-copy], button.copy[data-copy-text]')){" "btn.addEventListener('click', async () => {" @@ -4185,6 +4268,7 @@ def _format_setting(value: object) -> str: "}" "searchEl?.addEventListener('input', applyFilters);" "filterEl?.addEventListener('change', applyFilters);" + "showNoOpsEl?.addEventListener('change', applyFilters);" "computeStats();" "applyFilters();" "})();" diff --git a/duplicate_consolidation_executor.py b/duplicate_consolidation_executor.py index 2ed9a52..d5ca1ec 100644 --- a/duplicate_consolidation_executor.py +++ b/duplicate_consolidation_executor.py @@ -654,6 +654,7 @@ def _check_cancel(stage: str) -> None: loser_disposition: Dict[str, str] = {} artwork_actions: List[ArtworkDirective] = [] planned_tags: Dict[str, Dict[str, object]] = {} + playlist_rewrite_map: Dict[str, str] = {} path_to_group: Dict[str, str] = {} playlist_targets: Dict[str, str] = {} playlist_base_dirs: Dict[str, str] = {} @@ -671,7 +672,7 @@ def _find_loser_references( continue playlist_dir = (playlist_base_dirs or {}).get(playlist) or os.path.dirname(working_path) normalized = {_normalize_playlist_entry(ln, playlist_dir) for ln in lines if ln} - hits = sorted([loser for loser in loser_to_winner.keys() if loser in normalized]) + hits = sorted([loser for loser in playlist_rewrite_map.keys() if loser in normalized]) if hits: refs[playlist] = hits return refs @@ -693,13 +694,15 @@ def _find_loser_references( group_lookup = {g.group_id: g for g in plan.groups} for group in plan.groups: - planned_tags[group.winner_path] = dict(group.planned_winner_tags) + if group.metadata_changes: + planned_tags[group.winner_path] = dict(group.planned_winner_tags) for loser in group.losers: loser_to_winner[loser] = group.winner_path loser_disposition[loser] = group.loser_disposition.get(loser, "quarantine") path_to_group[loser] = group.group_id path_to_group[group.winner_path] = group.group_id artwork_actions.extend(group.artwork) + playlist_rewrite_map.update(group.playlist_rewrites) computed_signature = _compute_plan_signature(plan) _check_cancel("start") @@ -792,7 +795,9 @@ def _find_loser_references( ) raise RuntimeError("Execution blocked: review-required groups present.") - planned_operations = len(loser_to_winner) + len(artwork_actions) + len(planned_tags) + planned_operations = ( + len(loser_to_winner) + len(artwork_actions) + len(planned_tags) + len(playlist_rewrite_map) + ) if config.operation_limit and planned_operations > config.operation_limit and not config.confirm_operation_overage: success = False _record( @@ -847,7 +852,7 @@ def _find_loser_references( continue playlist_dir = os.path.dirname(playlist) normalized = {_normalize_playlist_entry(ln, playlist_dir) for ln in lines if ln} - affected = normalized & loser_to_winner.keys() + affected = normalized & playlist_rewrite_map.keys() if affected: impacted.append(playlist) playlist_groups[playlist] = {path_to_group.get(path, "") for path in affected if path in path_to_group} @@ -921,7 +926,7 @@ def _find_loser_references( _check_cancel("rewrite_playlists") target_playlist = playlist_targets.get(playlist, playlist) base_dir = playlist_base_dirs.get(playlist, os.path.dirname(target_playlist)) - replaced, new_lines = _rewrite_playlist(target_playlist, loser_to_winner, base_dir=base_dir) + replaced, new_lines = _rewrite_playlist(target_playlist, playlist_rewrite_map, base_dir=base_dir) if replaced == 0: playlist_results.append( PlaylistRewriteResult( diff --git a/tests/test_duplicate_consolidation_preview.py b/tests/test_duplicate_consolidation_preview.py index d22ad73..409969d 100644 --- a/tests/test_duplicate_consolidation_preview.py +++ b/tests/test_duplicate_consolidation_preview.py @@ -56,7 +56,7 @@ def test_plan_contains_metadata_and_quality_summary(tmp_path) -> None: assert group.metadata_changes # winner tags should be merged assert group.winner_quality["reasons"] # rationale present - assert group.playlist_impact.entries == len(group.losers) + assert group.playlist_impact.entries == 0 assert group.winner_current_tags # real tags captured assert group.current_tags[group.winner_path]["title"] == "Song A" assert group.chosen_artwork_source # artwork decision captured even if unchanged From 62c233b20f6217b0f00b500d1b9c2a9087d6e836 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 6 Jan 2026 19:39:21 -0500 Subject: [PATCH 296/606] Limit fingerprinting in-flight tasks --- simple_duplicate_finder.py | 44 +++++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/simple_duplicate_finder.py b/simple_duplicate_finder.py index cfbe8f5..faa722c 100644 --- a/simple_duplicate_finder.py +++ b/simple_duplicate_finder.py @@ -148,21 +148,45 @@ def compute(path: str) -> Tuple[Optional[int], Optional[str]]: total = len(audio_paths) file_data: List[Tuple[str, str]] = [] - futures: Dict[object, Tuple[int, str]] = {} + in_flight: Dict[object, Tuple[int, str]] = {} with ThreadPoolExecutor(max_workers=max_workers) as executor: - for idx, p in enumerate(audio_paths, 1): - if cancel_event and cancel_event.is_set(): - break + effective_workers = max_workers if max_workers is not None else executor._max_workers + max_in_flight = max(1, effective_workers * 2) + + def submit_task(idx: int, path: str) -> None: if progress_callback: - progress_callback("fp_start", idx, total, p) + progress_callback("fp_start", idx, total, path) fut = executor.submit( - get_fingerprint, p, db_path, compute, log_callback=log_callback + get_fingerprint, path, db_path, compute, log_callback=log_callback ) - futures[fut] = (idx, p) + in_flight[fut] = (idx, path) + + for idx, p in enumerate(audio_paths, 1): + if cancel_event and cancel_event.is_set(): + break + submit_task(idx, p) + while len(in_flight) >= max_in_flight: + fut = next(as_completed(in_flight)) + idx, p = in_flight.pop(fut) + if cancel_event and cancel_event.is_set(): + break + fp = fut.result() + if progress_callback: + progress_callback("fp_end", idx, total, p) + if fp: + log_callback(f"\u2713 Fingerprinted {p}") + show_pref = fp[:prefix_len] if (prefix_len and prefix_len > 0) else "" + _dlog("FP", f"prefix={show_pref} value={fp}") + file_data.append((p, fp)) + _dlog("GROUP", f"added file_data {p}") + else: + log_callback(f"\u2717 No fingerprint for {p}") + if cancel_event and cancel_event.is_set(): + break - for fut in as_completed(futures): - idx, p = futures[fut] + for fut in as_completed(in_flight): + idx, p = in_flight[fut] if cancel_event and cancel_event.is_set(): break fp = fut.result() @@ -178,7 +202,7 @@ def compute(path: str) -> Tuple[Optional[int], Optional[str]]: log_callback(f"\u2717 No fingerprint for {p}") if cancel_event and cancel_event.is_set(): - for fut in futures: + for fut in in_flight: fut.cancel() groups: List[Dict[str, object]] = [] From 31b1775f1a812f0c5ad31add33aadbff7f81d989 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 6 Jan 2026 19:52:08 -0500 Subject: [PATCH 297/606] Handle fingerprint task failures in duplicate finder --- simple_duplicate_finder.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/simple_duplicate_finder.py b/simple_duplicate_finder.py index faa722c..b6e0e8c 100644 --- a/simple_duplicate_finder.py +++ b/simple_duplicate_finder.py @@ -171,7 +171,11 @@ def submit_task(idx: int, path: str) -> None: idx, p = in_flight.pop(fut) if cancel_event and cancel_event.is_set(): break - fp = fut.result() + try: + fp = fut.result() + except Exception as exc: + fp = None + log_callback(f"! Fingerprint task failed for {p}: {exc}") if progress_callback: progress_callback("fp_end", idx, total, p) if fp: @@ -189,7 +193,11 @@ def submit_task(idx: int, path: str) -> None: idx, p = in_flight[fut] if cancel_event and cancel_event.is_set(): break - fp = fut.result() + try: + fp = fut.result() + except Exception as exc: + fp = None + log_callback(f"! Fingerprint task failed for {p}: {exc}") if progress_callback: progress_callback("fp_end", idx, total, p) if fp: From ce34f6d6093df76c8f8637bc61edbc95fffa1a44 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 6 Jan 2026 20:03:26 -0500 Subject: [PATCH 298/606] Stream duplicate fingerprint processing --- simple_duplicate_finder.py | 156 ++++++++++++++++++------------------- 1 file changed, 78 insertions(+), 78 deletions(-) diff --git a/simple_duplicate_finder.py b/simple_duplicate_finder.py index b6e0e8c..c146c30 100644 --- a/simple_duplicate_finder.py +++ b/simple_duplicate_finder.py @@ -2,7 +2,7 @@ import time import threading from concurrent.futures import ThreadPoolExecutor, as_completed -from typing import Callable, List, Tuple, Dict, Optional +from typing import Callable, List, Tuple, Dict, Optional, Iterator from fingerprint_cache import get_fingerprint from near_duplicate_detector import fingerprint_distance @@ -77,8 +77,8 @@ def _walk_audio_files( root: str, progress_callback: Optional[Callable[[str, int, int, str], None]] = None, cancel_event: Optional[threading.Event] = None, -) -> List[str]: - paths: List[str] = [] +) -> Iterator[str]: + found = 0 for dirpath, _dirs, files in os.walk(root): _dlog("WALK", f"enter {dirpath}") rel = os.path.relpath(dirpath, root) @@ -91,18 +91,18 @@ def _walk_audio_files( if ext in SUPPORTED_EXTS: path = os.path.join(dirpath, fname) _dlog("WALK", f"match {path}") - paths.append(path) + found += 1 if progress_callback: - progress_callback("walk", len(paths), 0, path) + progress_callback("walk", found, 0, path) + yield path if cancel_event and cancel_event.is_set(): - return paths + return else: _dlog("WALK", f"skip file {fname}") if cancel_event and cancel_event.is_set(): - return paths + return if cancel_event and cancel_event.is_set(): - return paths - return paths + return def find_duplicates( @@ -142,17 +142,53 @@ def compute(path: str) -> Tuple[Optional[int], Optional[str]]: missing_fp += 1 return duration, fp - audio_paths = _walk_audio_files(root, progress_callback, cancel_event) - if cancel_event and cancel_event.is_set(): - return [], missing_fp - - total = len(audio_paths) - file_data: List[Tuple[str, str]] = [] + total = 0 # Unknown upfront; progress callbacks receive 0 for total. in_flight: Dict[object, Tuple[int, str]] = {} + cancelled = False + + groups: List[Dict[str, object]] = [] + prefix_map: Dict[str, List[Dict[str, object]]] = {} + use_prefix = prefix_len is not None and prefix_len > 0 + + def handle_fingerprint(path: str, fp: str) -> None: + prefix = fp[:prefix_len] if use_prefix else "" + log_callback(f"[GROUP] path={path}, prefix={prefix}") + cand_groups: List[Tuple[Dict[str, object], str]] = [] + if use_prefix: + for key, groups_for_key in prefix_map.items(): + if prefix_distance(prefix, key) <= PREFIX_THRESHOLD: + for g in groups_for_key: + cand_groups.append((g, key)) + else: + for g in prefix_map.get(prefix, []): + cand_groups.append((g, prefix)) + _dlog("GROUP", f"file {path} -> prefix {prefix}") + _dlog("GROUP", f"{len(cand_groups)} groups for prefix") + placed = False + for g, key in cand_groups: + dist = fingerprint_distance(fp, g["fp"]) + _dlog("DIST", f"{path} vs {g['paths'][0]} dist={dist:.3f} threshold={threshold}") + log_callback( + f"[DIST] {path} \u2194 {g['paths'][0]} distance={dist:.4f} (thr={threshold:.4f})" + ) + if dist <= threshold: + if key != prefix: + log_callback( + f"[FUZZY] {path} matched prefix {key} (dist={prefix_distance(prefix, key)})" + ) + g["paths"].append(path) + _dlog("GROUP", f"added to existing group with {g['paths'][0]}") + placed = True + break + if not placed: + g = {"fp": fp, "paths": [path]} + prefix_map.setdefault(prefix, []).append(g) + groups.append(g) + _dlog("GROUP", f"new group for prefix {prefix}") with ThreadPoolExecutor(max_workers=max_workers) as executor: effective_workers = max_workers if max_workers is not None else executor._max_workers - max_in_flight = max(1, effective_workers * 2) + max_in_flight = max(1, effective_workers * 4) def submit_task(idx: int, path: str) -> None: if progress_callback: @@ -162,14 +198,16 @@ def submit_task(idx: int, path: str) -> None: ) in_flight[fut] = (idx, path) - for idx, p in enumerate(audio_paths, 1): + for idx, p in enumerate(_walk_audio_files(root, progress_callback, cancel_event), 1): if cancel_event and cancel_event.is_set(): + cancelled = True break submit_task(idx, p) while len(in_flight) >= max_in_flight: fut = next(as_completed(in_flight)) idx, p = in_flight.pop(fut) if cancel_event and cancel_event.is_set(): + cancelled = True break try: fp = fut.result() @@ -182,76 +220,38 @@ def submit_task(idx: int, path: str) -> None: log_callback(f"\u2713 Fingerprinted {p}") show_pref = fp[:prefix_len] if (prefix_len and prefix_len > 0) else "" _dlog("FP", f"prefix={show_pref} value={fp}") - file_data.append((p, fp)) - _dlog("GROUP", f"added file_data {p}") + handle_fingerprint(p, fp) else: log_callback(f"\u2717 No fingerprint for {p}") if cancel_event and cancel_event.is_set(): + cancelled = True break - for fut in as_completed(in_flight): - idx, p = in_flight[fut] - if cancel_event and cancel_event.is_set(): - break - try: - fp = fut.result() - except Exception as exc: - fp = None - log_callback(f"! Fingerprint task failed for {p}: {exc}") - if progress_callback: - progress_callback("fp_end", idx, total, p) - if fp: - log_callback(f"\u2713 Fingerprinted {p}") - show_pref = fp[:prefix_len] if (prefix_len and prefix_len > 0) else "" - _dlog("FP", f"prefix={show_pref} value={fp}") - file_data.append((p, fp)) - _dlog("GROUP", f"added file_data {p}") - else: - log_callback(f"\u2717 No fingerprint for {p}") + if not cancelled: + for fut in as_completed(in_flight): + idx, p = in_flight[fut] + if cancel_event and cancel_event.is_set(): + cancelled = True + break + try: + fp = fut.result() + except Exception as exc: + fp = None + log_callback(f"! Fingerprint task failed for {p}: {exc}") + if progress_callback: + progress_callback("fp_end", idx, total, p) + if fp: + log_callback(f"\u2713 Fingerprinted {p}") + show_pref = fp[:prefix_len] if (prefix_len and prefix_len > 0) else "" + _dlog("FP", f"prefix={show_pref} value={fp}") + handle_fingerprint(p, fp) + else: + log_callback(f"\u2717 No fingerprint for {p}") - if cancel_event and cancel_event.is_set(): + if cancel_event and cancel_event.is_set() or cancelled: for fut in in_flight: fut.cancel() - groups: List[Dict[str, object]] = [] - prefix_map: Dict[str, List[Dict[str, object]]] = {} - use_prefix = prefix_len is not None and prefix_len > 0 - for path, fp in file_data: - prefix = fp[:prefix_len] if use_prefix else "" - log_callback(f"[GROUP] path={path}, prefix={prefix}") - cand_groups: List[Tuple[Dict[str, object], str]] = [] - if use_prefix: - for key, groups_for_key in prefix_map.items(): - if prefix_distance(prefix, key) <= PREFIX_THRESHOLD: - for g in groups_for_key: - cand_groups.append((g, key)) - else: - for g in prefix_map.get(prefix, []): - cand_groups.append((g, prefix)) - _dlog("GROUP", f"file {path} -> prefix {prefix}") - _dlog("GROUP", f"{len(cand_groups)} groups for prefix") - placed = False - for g, key in cand_groups: - dist = fingerprint_distance(fp, g["fp"]) - _dlog("DIST", f"{path} vs {g['paths'][0]} dist={dist:.3f} threshold={threshold}") - log_callback( - f"[DIST] {path} \u2194 {g['paths'][0]} distance={dist:.4f} (thr={threshold:.4f})" - ) - if dist <= threshold: - if key != prefix: - log_callback( - f"[FUZZY] {path} matched prefix {key} (dist={prefix_distance(prefix, key)})" - ) - g["paths"].append(path) - _dlog("GROUP", f"added to existing group with {g['paths'][0]}") - placed = True - break - if not placed: - g = {"fp": fp, "paths": [path]} - prefix_map.setdefault(prefix, []).append(g) - groups.append(g) - _dlog("GROUP", f"new group for prefix {prefix}") - duplicates: List[Tuple[str, str]] = [] for g in groups: if cancel_event and cancel_event.is_set(): From bb82636f2f54cc2c8125c3095e7b30fa94ff22e0 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 6 Jan 2026 20:13:21 -0500 Subject: [PATCH 299/606] Add WAL-backed fingerprint writer --- fingerprint_cache.py | 221 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 191 insertions(+), 30 deletions(-) diff --git a/fingerprint_cache.py b/fingerprint_cache.py index 1bbb348..cf8bc3a 100644 --- a/fingerprint_cache.py +++ b/fingerprint_cache.py @@ -1,5 +1,7 @@ import os +import queue import sqlite3 +import threading import time from typing import Callable, Dict, Optional from utils.path_helpers import ensure_long_path @@ -18,9 +20,13 @@ def _dlog(label: str, msg: str, cb: Optional[Callable[[str], None]] = None) -> N print(line) -def _ensure_db(db_path: str) -> sqlite3.Connection: - os.makedirs(os.path.dirname(db_path), exist_ok=True) - conn = sqlite3.connect(db_path) +_writer_lock = threading.Lock() +_writer: "FingerprintWriter | None" = None +_writer_db_path: str | None = None + + +def _initialize_db(conn: sqlite3.Connection) -> None: + conn.execute("PRAGMA journal_mode=WAL") conn.execute( """ CREATE TABLE IF NOT EXISTS fingerprints ( @@ -34,9 +40,7 @@ def _ensure_db(db_path: str) -> sqlite3.Connection: ) # --- Handle schema upgrades ------------------------------------------- - cols = { - row[1] for row in conn.execute("PRAGMA table_info(fingerprints)") - } + cols = {row[1] for row in conn.execute("PRAGMA table_info(fingerprints)")} if "mtime" not in cols: conn.execute("ALTER TABLE fingerprints ADD COLUMN mtime REAL") if "size" not in cols: @@ -47,6 +51,145 @@ def _ensure_db(db_path: str) -> sqlite3.Connection: conn.execute("ALTER TABLE fingerprints ADD COLUMN fingerprint TEXT") conn.commit() + +class FingerprintWriter: + def __init__( + self, + db_path: str, + *, + batch_size: int = 50, + flush_interval: float = 1.0, + ) -> None: + self._db_path = db_path + self._batch_size = batch_size + self._flush_interval = flush_interval + self._queue: "queue.Queue[tuple[str, object]]" = queue.Queue() + self._thread = threading.Thread( + target=self._run, + name="FingerprintCacheWriter", + daemon=True, + ) + self._thread.start() + + def enqueue( + self, + path: str, + mtime: float, + size: int, + duration: int | None, + fingerprint: str, + ) -> None: + self._queue.put(("write", (path, mtime, size, duration, fingerprint))) + + def flush(self, timeout: float | None = None) -> None: + event = threading.Event() + self._queue.put(("flush", event)) + event.wait(timeout=timeout) + + def shutdown(self) -> None: + event = threading.Event() + self._queue.put(("shutdown", event)) + self._thread.join() + event.wait(timeout=1.0) + + def _run(self) -> None: + conn: sqlite3.Connection | None = None + try: + conn = sqlite3.connect(self._db_path, check_same_thread=False) + _initialize_db(conn) + pending: list[tuple[str, float, int, int | None, str]] = [] + last_flush = time.monotonic() + while True: + timeout = self._flush_interval - (time.monotonic() - last_flush) + if timeout < 0: + timeout = 0 + try: + kind, payload = self._queue.get(timeout=timeout) + except queue.Empty: + kind = "" + payload = None + + if kind == "write": + pending.append(payload) # type: ignore[arg-type] + if len(pending) >= self._batch_size: + self._flush_pending(conn, pending) + pending.clear() + last_flush = time.monotonic() + elif kind == "flush": + self._flush_pending(conn, pending) + pending.clear() + last_flush = time.monotonic() + if isinstance(payload, threading.Event): + payload.set() + elif kind == "shutdown": + self._flush_pending(conn, pending) + pending.clear() + self._drain_remaining(conn) + if isinstance(payload, threading.Event): + payload.set() + break + elif kind == "": + if pending: + self._flush_pending(conn, pending) + pending.clear() + last_flush = time.monotonic() + finally: + if conn is not None: + conn.close() + + def _flush_pending( + self, + conn: sqlite3.Connection, + pending: list[tuple[str, float, int, int | None, str]], + ) -> None: + if not pending: + return + for attempt in range(3): + try: + conn.executemany( + "INSERT OR REPLACE INTO fingerprints (path, mtime, size, duration, fingerprint) VALUES (?, ?, ?, ?, ?)", + pending, + ) + conn.commit() + return + except sqlite3.OperationalError as exc: + if "locked" not in str(exc).lower() or attempt >= 2: + _dlog("FP", f"cache write failed: {exc}") + return + time.sleep(0.05) + + def _drain_remaining(self, conn: sqlite3.Connection) -> None: + pending: list[tuple[str, float, int, int | None, str]] = [] + while True: + try: + kind, payload = self._queue.get_nowait() + except queue.Empty: + break + if kind == "write": + pending.append(payload) # type: ignore[arg-type] + elif kind in {"flush", "shutdown"} and isinstance(payload, threading.Event): + payload.set() + if pending: + self._flush_pending(conn, pending) + + +def _get_writer(db_path: str) -> FingerprintWriter: + global _writer, _writer_db_path + with _writer_lock: + if _writer is None or _writer_db_path != db_path: + if _writer is not None: + _writer.shutdown() + os.makedirs(os.path.dirname(db_path), exist_ok=True) + _writer_db_path = db_path + _writer = FingerprintWriter(db_path) + return _writer + + +def _open_readonly_connection(db_path: str) -> sqlite3.Connection | None: + if not os.path.exists(db_path): + return None + conn = sqlite3.connect(db_path) + conn.execute("PRAGMA query_only = ON") return conn @@ -65,7 +208,7 @@ def get_fingerprint( path = ensure_long_path(path) os.makedirs(os.path.dirname(db_path), exist_ok=True) - conn = _ensure_db(db_path) + conn = _open_readonly_connection(db_path) try: mtime = os.path.getmtime(path) size = os.path.getsize(path) @@ -73,18 +216,22 @@ def get_fingerprint( log_callback(f"! Could not stat {path}: {e}") trace["source"] = "stat_error" trace["error"] = str(e) - conn.close() + if conn is not None: + conn.close() return None - row = conn.execute( - "SELECT mtime, size, fingerprint FROM fingerprints WHERE path=?", - (path,), - ).fetchone() + row = None + if conn is not None: + row = conn.execute( + "SELECT mtime, size, fingerprint FROM fingerprints WHERE path=?", + (path,), + ).fetchone() cached_mtime = row[0] if row else None cached_size = row[1] if row else None if row and abs(cached_mtime - mtime) < 1e-6 and int(cached_size or 0) == int(size): fp = row[2] _dlog("FP", f"cache hit {path}", log_callback) - conn.close() + if conn is not None: + conn.close() if isinstance(fp, (bytes, bytearray)): try: fp = fp.decode("utf-8") @@ -105,12 +252,9 @@ def get_fingerprint( duration, fp_hash = compute_func(path) if fp_hash is not None: _dlog("FP", f"computed fingerprint {fp_hash} prefix={fp_hash[:16]}", log_callback) - conn.execute( - "INSERT OR REPLACE INTO fingerprints (path, mtime, size, duration, fingerprint) VALUES (?, ?, ?, ?, ?)", - (path, mtime, size, duration, fp_hash), - ) - conn.commit() - conn.close() + _get_writer(db_path).enqueue(path, mtime, size, duration, fp_hash) + if conn is not None: + conn.close() if fp_hash: trace["source"] = "computed" trace["error"] = "" @@ -140,7 +284,11 @@ def get_cached_fingerprint( for attempt in range(retries): conn: sqlite3.Connection | None = None try: - conn = _ensure_db(db_path) + conn = _open_readonly_connection(db_path) + if conn is None: + trace["source"] = "missing" + trace["error"] = "" + return None try: mtime = os.path.getmtime(path) size = os.path.getsize(path) @@ -192,6 +340,7 @@ def store_fingerprint( *, retries: int = 3, retry_delay: float = 0.05, + flush: bool = False, ) -> bool: """Persist a fingerprint in the cache without computing it.""" if fingerprint is None: @@ -202,35 +351,47 @@ def store_fingerprint( path = ensure_long_path(path) os.makedirs(os.path.dirname(db_path), exist_ok=True) for attempt in range(retries): - conn: sqlite3.Connection | None = None try: - conn = _ensure_db(db_path) try: mtime = os.path.getmtime(path) size = os.path.getsize(path) except OSError as e: log_callback(f"! Could not stat {path}: {e}") return False - conn.execute( - "INSERT OR REPLACE INTO fingerprints (path, mtime, size, duration, fingerprint) VALUES (?, ?, ?, ?, ?)", - (path, mtime, size, duration, fingerprint), - ) - conn.commit() + _get_writer(db_path).enqueue(path, mtime, size, duration, fingerprint) + if flush: + flush_fingerprint_writes(db_path) return True except sqlite3.OperationalError as e: if "locked" not in str(e).lower() or attempt >= retries - 1: log_callback(f"! Fingerprint cache write failed: {e}") return False time.sleep(retry_delay) - finally: - if conn is not None: - conn.close() return False +def flush_fingerprint_writes(db_path: str, timeout: float | None = None) -> None: + """Force a flush of queued fingerprint writes for immediate persistence.""" + if not os.path.exists(db_path): + return + _get_writer(db_path).flush(timeout=timeout) + + +def shutdown_fingerprint_writer() -> None: + """Drain queued fingerprint writes; call during application shutdown (e.g., exit handler).""" + global _writer, _writer_db_path + with _writer_lock: + if _writer is None: + return + _writer.shutdown() + _writer = None + _writer_db_path = None + + def flush_cache(db_path: str) -> None: if not os.path.exists(db_path): return + shutdown_fingerprint_writer() try: os.remove(db_path) except Exception: From 171df870b6015b21555f8dbcf2c63b690c40147b Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 6 Jan 2026 20:14:09 -0500 Subject: [PATCH 300/606] Revert "Add WAL-backed background writer for fingerprint cache" --- fingerprint_cache.py | 221 ++++++------------------------------------- 1 file changed, 30 insertions(+), 191 deletions(-) diff --git a/fingerprint_cache.py b/fingerprint_cache.py index cf8bc3a..1bbb348 100644 --- a/fingerprint_cache.py +++ b/fingerprint_cache.py @@ -1,7 +1,5 @@ import os -import queue import sqlite3 -import threading import time from typing import Callable, Dict, Optional from utils.path_helpers import ensure_long_path @@ -20,13 +18,9 @@ def _dlog(label: str, msg: str, cb: Optional[Callable[[str], None]] = None) -> N print(line) -_writer_lock = threading.Lock() -_writer: "FingerprintWriter | None" = None -_writer_db_path: str | None = None - - -def _initialize_db(conn: sqlite3.Connection) -> None: - conn.execute("PRAGMA journal_mode=WAL") +def _ensure_db(db_path: str) -> sqlite3.Connection: + os.makedirs(os.path.dirname(db_path), exist_ok=True) + conn = sqlite3.connect(db_path) conn.execute( """ CREATE TABLE IF NOT EXISTS fingerprints ( @@ -40,7 +34,9 @@ def _initialize_db(conn: sqlite3.Connection) -> None: ) # --- Handle schema upgrades ------------------------------------------- - cols = {row[1] for row in conn.execute("PRAGMA table_info(fingerprints)")} + cols = { + row[1] for row in conn.execute("PRAGMA table_info(fingerprints)") + } if "mtime" not in cols: conn.execute("ALTER TABLE fingerprints ADD COLUMN mtime REAL") if "size" not in cols: @@ -51,145 +47,6 @@ def _initialize_db(conn: sqlite3.Connection) -> None: conn.execute("ALTER TABLE fingerprints ADD COLUMN fingerprint TEXT") conn.commit() - -class FingerprintWriter: - def __init__( - self, - db_path: str, - *, - batch_size: int = 50, - flush_interval: float = 1.0, - ) -> None: - self._db_path = db_path - self._batch_size = batch_size - self._flush_interval = flush_interval - self._queue: "queue.Queue[tuple[str, object]]" = queue.Queue() - self._thread = threading.Thread( - target=self._run, - name="FingerprintCacheWriter", - daemon=True, - ) - self._thread.start() - - def enqueue( - self, - path: str, - mtime: float, - size: int, - duration: int | None, - fingerprint: str, - ) -> None: - self._queue.put(("write", (path, mtime, size, duration, fingerprint))) - - def flush(self, timeout: float | None = None) -> None: - event = threading.Event() - self._queue.put(("flush", event)) - event.wait(timeout=timeout) - - def shutdown(self) -> None: - event = threading.Event() - self._queue.put(("shutdown", event)) - self._thread.join() - event.wait(timeout=1.0) - - def _run(self) -> None: - conn: sqlite3.Connection | None = None - try: - conn = sqlite3.connect(self._db_path, check_same_thread=False) - _initialize_db(conn) - pending: list[tuple[str, float, int, int | None, str]] = [] - last_flush = time.monotonic() - while True: - timeout = self._flush_interval - (time.monotonic() - last_flush) - if timeout < 0: - timeout = 0 - try: - kind, payload = self._queue.get(timeout=timeout) - except queue.Empty: - kind = "" - payload = None - - if kind == "write": - pending.append(payload) # type: ignore[arg-type] - if len(pending) >= self._batch_size: - self._flush_pending(conn, pending) - pending.clear() - last_flush = time.monotonic() - elif kind == "flush": - self._flush_pending(conn, pending) - pending.clear() - last_flush = time.monotonic() - if isinstance(payload, threading.Event): - payload.set() - elif kind == "shutdown": - self._flush_pending(conn, pending) - pending.clear() - self._drain_remaining(conn) - if isinstance(payload, threading.Event): - payload.set() - break - elif kind == "": - if pending: - self._flush_pending(conn, pending) - pending.clear() - last_flush = time.monotonic() - finally: - if conn is not None: - conn.close() - - def _flush_pending( - self, - conn: sqlite3.Connection, - pending: list[tuple[str, float, int, int | None, str]], - ) -> None: - if not pending: - return - for attempt in range(3): - try: - conn.executemany( - "INSERT OR REPLACE INTO fingerprints (path, mtime, size, duration, fingerprint) VALUES (?, ?, ?, ?, ?)", - pending, - ) - conn.commit() - return - except sqlite3.OperationalError as exc: - if "locked" not in str(exc).lower() or attempt >= 2: - _dlog("FP", f"cache write failed: {exc}") - return - time.sleep(0.05) - - def _drain_remaining(self, conn: sqlite3.Connection) -> None: - pending: list[tuple[str, float, int, int | None, str]] = [] - while True: - try: - kind, payload = self._queue.get_nowait() - except queue.Empty: - break - if kind == "write": - pending.append(payload) # type: ignore[arg-type] - elif kind in {"flush", "shutdown"} and isinstance(payload, threading.Event): - payload.set() - if pending: - self._flush_pending(conn, pending) - - -def _get_writer(db_path: str) -> FingerprintWriter: - global _writer, _writer_db_path - with _writer_lock: - if _writer is None or _writer_db_path != db_path: - if _writer is not None: - _writer.shutdown() - os.makedirs(os.path.dirname(db_path), exist_ok=True) - _writer_db_path = db_path - _writer = FingerprintWriter(db_path) - return _writer - - -def _open_readonly_connection(db_path: str) -> sqlite3.Connection | None: - if not os.path.exists(db_path): - return None - conn = sqlite3.connect(db_path) - conn.execute("PRAGMA query_only = ON") return conn @@ -208,7 +65,7 @@ def get_fingerprint( path = ensure_long_path(path) os.makedirs(os.path.dirname(db_path), exist_ok=True) - conn = _open_readonly_connection(db_path) + conn = _ensure_db(db_path) try: mtime = os.path.getmtime(path) size = os.path.getsize(path) @@ -216,22 +73,18 @@ def get_fingerprint( log_callback(f"! Could not stat {path}: {e}") trace["source"] = "stat_error" trace["error"] = str(e) - if conn is not None: - conn.close() + conn.close() return None - row = None - if conn is not None: - row = conn.execute( - "SELECT mtime, size, fingerprint FROM fingerprints WHERE path=?", - (path,), - ).fetchone() + row = conn.execute( + "SELECT mtime, size, fingerprint FROM fingerprints WHERE path=?", + (path,), + ).fetchone() cached_mtime = row[0] if row else None cached_size = row[1] if row else None if row and abs(cached_mtime - mtime) < 1e-6 and int(cached_size or 0) == int(size): fp = row[2] _dlog("FP", f"cache hit {path}", log_callback) - if conn is not None: - conn.close() + conn.close() if isinstance(fp, (bytes, bytearray)): try: fp = fp.decode("utf-8") @@ -252,9 +105,12 @@ def get_fingerprint( duration, fp_hash = compute_func(path) if fp_hash is not None: _dlog("FP", f"computed fingerprint {fp_hash} prefix={fp_hash[:16]}", log_callback) - _get_writer(db_path).enqueue(path, mtime, size, duration, fp_hash) - if conn is not None: - conn.close() + conn.execute( + "INSERT OR REPLACE INTO fingerprints (path, mtime, size, duration, fingerprint) VALUES (?, ?, ?, ?, ?)", + (path, mtime, size, duration, fp_hash), + ) + conn.commit() + conn.close() if fp_hash: trace["source"] = "computed" trace["error"] = "" @@ -284,11 +140,7 @@ def get_cached_fingerprint( for attempt in range(retries): conn: sqlite3.Connection | None = None try: - conn = _open_readonly_connection(db_path) - if conn is None: - trace["source"] = "missing" - trace["error"] = "" - return None + conn = _ensure_db(db_path) try: mtime = os.path.getmtime(path) size = os.path.getsize(path) @@ -340,7 +192,6 @@ def store_fingerprint( *, retries: int = 3, retry_delay: float = 0.05, - flush: bool = False, ) -> bool: """Persist a fingerprint in the cache without computing it.""" if fingerprint is None: @@ -351,47 +202,35 @@ def store_fingerprint( path = ensure_long_path(path) os.makedirs(os.path.dirname(db_path), exist_ok=True) for attempt in range(retries): + conn: sqlite3.Connection | None = None try: + conn = _ensure_db(db_path) try: mtime = os.path.getmtime(path) size = os.path.getsize(path) except OSError as e: log_callback(f"! Could not stat {path}: {e}") return False - _get_writer(db_path).enqueue(path, mtime, size, duration, fingerprint) - if flush: - flush_fingerprint_writes(db_path) + conn.execute( + "INSERT OR REPLACE INTO fingerprints (path, mtime, size, duration, fingerprint) VALUES (?, ?, ?, ?, ?)", + (path, mtime, size, duration, fingerprint), + ) + conn.commit() return True except sqlite3.OperationalError as e: if "locked" not in str(e).lower() or attempt >= retries - 1: log_callback(f"! Fingerprint cache write failed: {e}") return False time.sleep(retry_delay) + finally: + if conn is not None: + conn.close() return False -def flush_fingerprint_writes(db_path: str, timeout: float | None = None) -> None: - """Force a flush of queued fingerprint writes for immediate persistence.""" - if not os.path.exists(db_path): - return - _get_writer(db_path).flush(timeout=timeout) - - -def shutdown_fingerprint_writer() -> None: - """Drain queued fingerprint writes; call during application shutdown (e.g., exit handler).""" - global _writer, _writer_db_path - with _writer_lock: - if _writer is None: - return - _writer.shutdown() - _writer = None - _writer_db_path = None - - def flush_cache(db_path: str) -> None: if not os.path.exists(db_path): return - shutdown_fingerprint_writer() try: os.remove(db_path) except Exception: From 9feff22df7daf7185aeac4bb379819eb2bdf1b1b Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 6 Jan 2026 20:17:54 -0500 Subject: [PATCH 301/606] Restore fingerprint cache ensure_db helper --- fingerprint_cache.py | 229 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 199 insertions(+), 30 deletions(-) diff --git a/fingerprint_cache.py b/fingerprint_cache.py index 1bbb348..c7975db 100644 --- a/fingerprint_cache.py +++ b/fingerprint_cache.py @@ -1,5 +1,7 @@ import os +import queue import sqlite3 +import threading import time from typing import Callable, Dict, Optional from utils.path_helpers import ensure_long_path @@ -18,9 +20,13 @@ def _dlog(label: str, msg: str, cb: Optional[Callable[[str], None]] = None) -> N print(line) -def _ensure_db(db_path: str) -> sqlite3.Connection: - os.makedirs(os.path.dirname(db_path), exist_ok=True) - conn = sqlite3.connect(db_path) +_writer_lock = threading.Lock() +_writer: "FingerprintWriter | None" = None +_writer_db_path: str | None = None + + +def _initialize_db(conn: sqlite3.Connection) -> None: + conn.execute("PRAGMA journal_mode=WAL") conn.execute( """ CREATE TABLE IF NOT EXISTS fingerprints ( @@ -34,9 +40,7 @@ def _ensure_db(db_path: str) -> sqlite3.Connection: ) # --- Handle schema upgrades ------------------------------------------- - cols = { - row[1] for row in conn.execute("PRAGMA table_info(fingerprints)") - } + cols = {row[1] for row in conn.execute("PRAGMA table_info(fingerprints)")} if "mtime" not in cols: conn.execute("ALTER TABLE fingerprints ADD COLUMN mtime REAL") if "size" not in cols: @@ -47,6 +51,153 @@ def _ensure_db(db_path: str) -> sqlite3.Connection: conn.execute("ALTER TABLE fingerprints ADD COLUMN fingerprint TEXT") conn.commit() + +class FingerprintWriter: + def __init__( + self, + db_path: str, + *, + batch_size: int = 50, + flush_interval: float = 1.0, + ) -> None: + self._db_path = db_path + self._batch_size = batch_size + self._flush_interval = flush_interval + self._queue: "queue.Queue[tuple[str, object]]" = queue.Queue() + self._thread = threading.Thread( + target=self._run, + name="FingerprintCacheWriter", + daemon=True, + ) + self._thread.start() + + def enqueue( + self, + path: str, + mtime: float, + size: int, + duration: int | None, + fingerprint: str, + ) -> None: + self._queue.put(("write", (path, mtime, size, duration, fingerprint))) + + def flush(self, timeout: float | None = None) -> None: + event = threading.Event() + self._queue.put(("flush", event)) + event.wait(timeout=timeout) + + def shutdown(self) -> None: + event = threading.Event() + self._queue.put(("shutdown", event)) + self._thread.join() + event.wait(timeout=1.0) + + def _run(self) -> None: + conn: sqlite3.Connection | None = None + try: + conn = sqlite3.connect(self._db_path, check_same_thread=False) + _initialize_db(conn) + pending: list[tuple[str, float, int, int | None, str]] = [] + last_flush = time.monotonic() + while True: + timeout = self._flush_interval - (time.monotonic() - last_flush) + if timeout < 0: + timeout = 0 + try: + kind, payload = self._queue.get(timeout=timeout) + except queue.Empty: + kind = "" + payload = None + + if kind == "write": + pending.append(payload) # type: ignore[arg-type] + if len(pending) >= self._batch_size: + self._flush_pending(conn, pending) + pending.clear() + last_flush = time.monotonic() + elif kind == "flush": + self._flush_pending(conn, pending) + pending.clear() + last_flush = time.monotonic() + if isinstance(payload, threading.Event): + payload.set() + elif kind == "shutdown": + self._flush_pending(conn, pending) + pending.clear() + self._drain_remaining(conn) + if isinstance(payload, threading.Event): + payload.set() + break + elif kind == "": + if pending: + self._flush_pending(conn, pending) + pending.clear() + last_flush = time.monotonic() + finally: + if conn is not None: + conn.close() + + def _flush_pending( + self, + conn: sqlite3.Connection, + pending: list[tuple[str, float, int, int | None, str]], + ) -> None: + if not pending: + return + for attempt in range(3): + try: + conn.executemany( + "INSERT OR REPLACE INTO fingerprints (path, mtime, size, duration, fingerprint) VALUES (?, ?, ?, ?, ?)", + pending, + ) + conn.commit() + return + except sqlite3.OperationalError as exc: + if "locked" not in str(exc).lower() or attempt >= 2: + _dlog("FP", f"cache write failed: {exc}") + return + time.sleep(0.05) + + def _drain_remaining(self, conn: sqlite3.Connection) -> None: + pending: list[tuple[str, float, int, int | None, str]] = [] + while True: + try: + kind, payload = self._queue.get_nowait() + except queue.Empty: + break + if kind == "write": + pending.append(payload) # type: ignore[arg-type] + elif kind in {"flush", "shutdown"} and isinstance(payload, threading.Event): + payload.set() + if pending: + self._flush_pending(conn, pending) + + +def _get_writer(db_path: str) -> FingerprintWriter: + global _writer, _writer_db_path + with _writer_lock: + if _writer is None or _writer_db_path != db_path: + if _writer is not None: + _writer.shutdown() + os.makedirs(os.path.dirname(db_path), exist_ok=True) + _writer_db_path = db_path + _writer = FingerprintWriter(db_path) + return _writer + + +def _open_readonly_connection(db_path: str) -> sqlite3.Connection | None: + if not os.path.exists(db_path): + return None + conn = sqlite3.connect(db_path) + conn.execute("PRAGMA query_only = ON") + return conn + + +def _ensure_db(db_path: str) -> sqlite3.Connection: + """Legacy helper for callers that need a ready connection.""" + os.makedirs(os.path.dirname(db_path), exist_ok=True) + conn = sqlite3.connect(db_path) + _initialize_db(conn) return conn @@ -65,7 +216,7 @@ def get_fingerprint( path = ensure_long_path(path) os.makedirs(os.path.dirname(db_path), exist_ok=True) - conn = _ensure_db(db_path) + conn = _open_readonly_connection(db_path) try: mtime = os.path.getmtime(path) size = os.path.getsize(path) @@ -73,18 +224,22 @@ def get_fingerprint( log_callback(f"! Could not stat {path}: {e}") trace["source"] = "stat_error" trace["error"] = str(e) - conn.close() + if conn is not None: + conn.close() return None - row = conn.execute( - "SELECT mtime, size, fingerprint FROM fingerprints WHERE path=?", - (path,), - ).fetchone() + row = None + if conn is not None: + row = conn.execute( + "SELECT mtime, size, fingerprint FROM fingerprints WHERE path=?", + (path,), + ).fetchone() cached_mtime = row[0] if row else None cached_size = row[1] if row else None if row and abs(cached_mtime - mtime) < 1e-6 and int(cached_size or 0) == int(size): fp = row[2] _dlog("FP", f"cache hit {path}", log_callback) - conn.close() + if conn is not None: + conn.close() if isinstance(fp, (bytes, bytearray)): try: fp = fp.decode("utf-8") @@ -105,12 +260,9 @@ def get_fingerprint( duration, fp_hash = compute_func(path) if fp_hash is not None: _dlog("FP", f"computed fingerprint {fp_hash} prefix={fp_hash[:16]}", log_callback) - conn.execute( - "INSERT OR REPLACE INTO fingerprints (path, mtime, size, duration, fingerprint) VALUES (?, ?, ?, ?, ?)", - (path, mtime, size, duration, fp_hash), - ) - conn.commit() - conn.close() + _get_writer(db_path).enqueue(path, mtime, size, duration, fp_hash) + if conn is not None: + conn.close() if fp_hash: trace["source"] = "computed" trace["error"] = "" @@ -140,7 +292,11 @@ def get_cached_fingerprint( for attempt in range(retries): conn: sqlite3.Connection | None = None try: - conn = _ensure_db(db_path) + conn = _open_readonly_connection(db_path) + if conn is None: + trace["source"] = "missing" + trace["error"] = "" + return None try: mtime = os.path.getmtime(path) size = os.path.getsize(path) @@ -192,6 +348,7 @@ def store_fingerprint( *, retries: int = 3, retry_delay: float = 0.05, + flush: bool = False, ) -> bool: """Persist a fingerprint in the cache without computing it.""" if fingerprint is None: @@ -202,35 +359,47 @@ def store_fingerprint( path = ensure_long_path(path) os.makedirs(os.path.dirname(db_path), exist_ok=True) for attempt in range(retries): - conn: sqlite3.Connection | None = None try: - conn = _ensure_db(db_path) try: mtime = os.path.getmtime(path) size = os.path.getsize(path) except OSError as e: log_callback(f"! Could not stat {path}: {e}") return False - conn.execute( - "INSERT OR REPLACE INTO fingerprints (path, mtime, size, duration, fingerprint) VALUES (?, ?, ?, ?, ?)", - (path, mtime, size, duration, fingerprint), - ) - conn.commit() + _get_writer(db_path).enqueue(path, mtime, size, duration, fingerprint) + if flush: + flush_fingerprint_writes(db_path) return True except sqlite3.OperationalError as e: if "locked" not in str(e).lower() or attempt >= retries - 1: log_callback(f"! Fingerprint cache write failed: {e}") return False time.sleep(retry_delay) - finally: - if conn is not None: - conn.close() return False +def flush_fingerprint_writes(db_path: str, timeout: float | None = None) -> None: + """Force a flush of queued fingerprint writes for immediate persistence.""" + if not os.path.exists(db_path): + return + _get_writer(db_path).flush(timeout=timeout) + + +def shutdown_fingerprint_writer() -> None: + """Drain queued fingerprint writes; call during application shutdown (e.g., exit handler).""" + global _writer, _writer_db_path + with _writer_lock: + if _writer is None: + return + _writer.shutdown() + _writer = None + _writer_db_path = None + + def flush_cache(db_path: str) -> None: if not os.path.exists(db_path): return + shutdown_fingerprint_writer() try: os.remove(db_path) except Exception: From c87208875d3f5ae39f43cb6f232535607b9d75bc Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 6 Jan 2026 20:29:48 -0500 Subject: [PATCH 302/606] Add preview timing and heartbeat logging --- main_gui.py | 202 ++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 139 insertions(+), 63 deletions(-) diff --git a/main_gui.py b/main_gui.py index 444673c..68a1f65 100644 --- a/main_gui.py +++ b/main_gui.py @@ -2803,69 +2803,82 @@ def _track_payload( return tracks def _generate_plan(self, write_preview: bool) -> None: - path = self._validate_library_root() - if not path: - return - - tracks = self._gather_tracks(path) - if not tracks: - messagebox.showwarning("No Tracks", "No audio tracks were found in the selected library.") - self._set_status("Idle", progress=0) - return - self._set_fingerprint_status("Grouping duplicates…") - self._set_status("Grouping…", progress=self._weighted_progress("grouping", 0)) - cfg = load_config() - threshold_settings = self._current_threshold_settings() - exact_threshold = threshold_settings["exact_duplicate_threshold"] - near_threshold = threshold_settings["near_duplicate_threshold"] - mixed_codec_boost = threshold_settings["mixed_codec_threshold_boost"] - fingerprint_settings = self._current_fingerprint_settings() - self._log_action( - "Duplicate scan thresholds: " - f"exact={exact_threshold:.3f}, near={near_threshold:.3f}, mixed_codec_boost={mixed_codec_boost:.3f}" - ) - self._log_action(f"Fingerprint settings: {fingerprint_settings}") - plan = build_consolidation_plan( - tracks, - exact_duplicate_threshold=exact_threshold, - near_duplicate_threshold=near_threshold, - mixed_codec_threshold_boost=mixed_codec_boost, - fingerprint_settings=fingerprint_settings, - threshold_settings=threshold_settings, - ) - self._set_status("Grouping…", progress=self._weighted_progress("grouping", 1)) - self._plan = plan - self.group_disposition_overrides.clear() - self._apply_deletion_mode() + timing_enabled = write_preview + if timing_enabled: + plan_start_ts = datetime.now().isoformat(timespec="seconds") + plan_start_time = time.monotonic() + self._log_action(f"Preview plan timing start: {plan_start_ts}") + try: + path = self._validate_library_root() + if not path: + return - docs_dir = os.path.join(path, "Docs") - os.makedirs(docs_dir, exist_ok=True) - if write_preview: - self._set_fingerprint_status("Generating preview…") - self._set_status("Preview…", progress=self._weighted_progress("preview", 0)) - json_path = os.path.join(docs_dir, "duplicate_preview.json") - export_consolidation_preview(plan, json_path) - self.preview_json_path = json_path - self._log_action(f"Audit JSON written to {json_path}") - html_path = os.path.join(docs_dir, "duplicate_preview.html") - export_consolidation_preview_html( - plan, - html_path, - show_artwork_variants=self.show_artwork_variants_var.get(), + tracks = self._gather_tracks(path) + if not tracks: + messagebox.showwarning("No Tracks", "No audio tracks were found in the selected library.") + self._set_status("Idle", progress=0) + return + self._set_fingerprint_status("Grouping duplicates…") + self._set_status("Grouping…", progress=self._weighted_progress("grouping", 0)) + cfg = load_config() + threshold_settings = self._current_threshold_settings() + exact_threshold = threshold_settings["exact_duplicate_threshold"] + near_threshold = threshold_settings["near_duplicate_threshold"] + mixed_codec_boost = threshold_settings["mixed_codec_threshold_boost"] + fingerprint_settings = self._current_fingerprint_settings() + self._log_action( + "Duplicate scan thresholds: " + f"exact={exact_threshold:.3f}, near={near_threshold:.3f}, mixed_codec_boost={mixed_codec_boost:.3f}" ) - self.preview_html_path = html_path - self._log_action(f"Preview HTML written to {html_path}") - self._set_status("Preview generated", progress=self._weighted_progress("preview", 1)) - else: - self.preview_json_path = None - self.preview_html_path = None - self._set_status("Plan ready", progress=100) + self._log_action(f"Fingerprint settings: {fingerprint_settings}") + plan = build_consolidation_plan( + tracks, + exact_duplicate_threshold=exact_threshold, + near_duplicate_threshold=near_threshold, + mixed_codec_threshold_boost=mixed_codec_boost, + fingerprint_settings=fingerprint_settings, + threshold_settings=threshold_settings, + ) + self._set_status("Grouping…", progress=self._weighted_progress("grouping", 1)) + self._plan = plan + self.group_disposition_overrides.clear() + self._apply_deletion_mode() - self._log_action( - f"Plan: {len(plan.groups)} groups, review required={plan.review_required_count}" - ) - if plan.review_required_count: - self._log_action("Review required groups will block execution unless overridden.") + docs_dir = os.path.join(path, "Docs") + os.makedirs(docs_dir, exist_ok=True) + if write_preview: + self._set_fingerprint_status("Generating preview…") + self._set_status("Preview…", progress=self._weighted_progress("preview", 0)) + json_path = os.path.join(docs_dir, "duplicate_preview.json") + export_consolidation_preview(plan, json_path) + self.preview_json_path = json_path + self._log_action(f"Audit JSON written to {json_path}") + html_path = os.path.join(docs_dir, "duplicate_preview.html") + export_consolidation_preview_html( + plan, + html_path, + show_artwork_variants=self.show_artwork_variants_var.get(), + ) + self.preview_html_path = html_path + self._log_action(f"Preview HTML written to {html_path}") + self._set_status("Preview generated", progress=self._weighted_progress("preview", 1)) + else: + self.preview_json_path = None + self.preview_html_path = None + self._set_status("Plan ready", progress=100) + + self._log_action( + f"Plan: {len(plan.groups)} groups, review required={plan.review_required_count}" + ) + if plan.review_required_count: + self._log_action("Review required groups will block execution unless overridden.") + finally: + if timing_enabled: + plan_elapsed = time.monotonic() - plan_start_time + plan_end_ts = datetime.now().isoformat(timespec="seconds") + self._log_action( + f"Preview plan timing end: {plan_end_ts} (elapsed {plan_elapsed:.2f}s)" + ) def _load_preview_plan(self, preview_path: str): try: @@ -2891,12 +2904,70 @@ def _handle_scan(self) -> None: self._log_action("Scan clicked") self._generate_plan(write_preview=False) + def _start_preview_heartbeat(self) -> None: + self._dup_preview_heartbeat_running = True + self._dup_preview_heartbeat_start = time.monotonic() + self._dup_preview_heartbeat_last_tick = None + self._dup_preview_heartbeat_count = 0 + self._log_action("Preview heartbeat started (100ms interval).") + self.after(int(self._dup_preview_heartbeat_interval * 1000), self._preview_heartbeat_tick) + + def _preview_heartbeat_tick(self) -> None: + if not self._dup_preview_heartbeat_running: + return + now = time.monotonic() + if self._dup_preview_heartbeat_last_tick is not None: + gap = now - self._dup_preview_heartbeat_last_tick + if gap > self._dup_preview_heartbeat_interval * 3: + self._log_action( + f"Preview heartbeat delayed by {gap:.2f}s while preview running." + ) + self._dup_preview_heartbeat_last_tick = now + self._dup_preview_heartbeat_count += 1 + self.after(int(self._dup_preview_heartbeat_interval * 1000), self._preview_heartbeat_tick) + + def _stop_preview_heartbeat(self) -> None: + if not self._dup_preview_heartbeat_running: + return + self._dup_preview_heartbeat_running = False + now = time.monotonic() + elapsed = 0.0 + if self._dup_preview_heartbeat_start is not None: + elapsed = now - self._dup_preview_heartbeat_start + if self._dup_preview_heartbeat_count == 0: + self._log_action( + f"Preview heartbeat did not fire during preview window ({elapsed:.2f}s)." + ) + elif ( + self._dup_preview_heartbeat_last_tick is not None + and now - self._dup_preview_heartbeat_last_tick + > self._dup_preview_heartbeat_interval * 3 + ): + stall = now - self._dup_preview_heartbeat_last_tick + self._log_action( + f"Preview heartbeat stalled for {stall:.2f}s before completion." + ) + self._log_action( + f"Preview heartbeat stopped after {elapsed:.2f}s " + f"({self._dup_preview_heartbeat_count} ticks)." + ) + def _handle_preview(self) -> None: self._reset_execute_confirmation() self._log_action("Preview clicked") - self._generate_plan(write_preview=True) - if self.preview_html_path: - self._open_preview() + start_ts = datetime.now().isoformat(timespec="seconds") + start_time = time.monotonic() + self._log_action(f"Preview timing start: {start_ts}") + self._start_preview_heartbeat() + try: + self._generate_plan(write_preview=True) + if self.preview_html_path: + self._open_preview() + finally: + elapsed = time.monotonic() - start_time + end_ts = datetime.now().isoformat(timespec="seconds") + self._stop_preview_heartbeat() + self._log_action(f"Preview timing end: {end_ts} (elapsed {elapsed:.2f}s)") def _handle_execute(self) -> None: if not self._execute_confirmation_pending: @@ -4246,6 +4317,11 @@ def __init__(self): self._preview_in_progress = False self._player_busy_item: str | None = None self._ignore_next_preview_finish = False + self._dup_preview_heartbeat_running = False + self._dup_preview_heartbeat_start: float | None = None + self._dup_preview_heartbeat_last_tick: float | None = None + self._dup_preview_heartbeat_count = 0 + self._dup_preview_heartbeat_interval = 0.1 # Player tab state self.player_tracks: list[dict[str, str]] = [] From b23ab1624ff90abe4fed882da70d10bf9b14ef8b Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 6 Jan 2026 20:34:58 -0500 Subject: [PATCH 303/606] Revert "Instrument duplicate preview with timing and heartbeat logging" --- main_gui.py | 202 ++++++++++++++++------------------------------------ 1 file changed, 63 insertions(+), 139 deletions(-) diff --git a/main_gui.py b/main_gui.py index 68a1f65..444673c 100644 --- a/main_gui.py +++ b/main_gui.py @@ -2803,82 +2803,69 @@ def _track_payload( return tracks def _generate_plan(self, write_preview: bool) -> None: - timing_enabled = write_preview - if timing_enabled: - plan_start_ts = datetime.now().isoformat(timespec="seconds") - plan_start_time = time.monotonic() - self._log_action(f"Preview plan timing start: {plan_start_ts}") - try: - path = self._validate_library_root() - if not path: - return - - tracks = self._gather_tracks(path) - if not tracks: - messagebox.showwarning("No Tracks", "No audio tracks were found in the selected library.") - self._set_status("Idle", progress=0) - return - self._set_fingerprint_status("Grouping duplicates…") - self._set_status("Grouping…", progress=self._weighted_progress("grouping", 0)) - cfg = load_config() - threshold_settings = self._current_threshold_settings() - exact_threshold = threshold_settings["exact_duplicate_threshold"] - near_threshold = threshold_settings["near_duplicate_threshold"] - mixed_codec_boost = threshold_settings["mixed_codec_threshold_boost"] - fingerprint_settings = self._current_fingerprint_settings() - self._log_action( - "Duplicate scan thresholds: " - f"exact={exact_threshold:.3f}, near={near_threshold:.3f}, mixed_codec_boost={mixed_codec_boost:.3f}" - ) - self._log_action(f"Fingerprint settings: {fingerprint_settings}") - plan = build_consolidation_plan( - tracks, - exact_duplicate_threshold=exact_threshold, - near_duplicate_threshold=near_threshold, - mixed_codec_threshold_boost=mixed_codec_boost, - fingerprint_settings=fingerprint_settings, - threshold_settings=threshold_settings, - ) - self._set_status("Grouping…", progress=self._weighted_progress("grouping", 1)) - self._plan = plan - self.group_disposition_overrides.clear() - self._apply_deletion_mode() + path = self._validate_library_root() + if not path: + return - docs_dir = os.path.join(path, "Docs") - os.makedirs(docs_dir, exist_ok=True) - if write_preview: - self._set_fingerprint_status("Generating preview…") - self._set_status("Preview…", progress=self._weighted_progress("preview", 0)) - json_path = os.path.join(docs_dir, "duplicate_preview.json") - export_consolidation_preview(plan, json_path) - self.preview_json_path = json_path - self._log_action(f"Audit JSON written to {json_path}") - html_path = os.path.join(docs_dir, "duplicate_preview.html") - export_consolidation_preview_html( - plan, - html_path, - show_artwork_variants=self.show_artwork_variants_var.get(), - ) - self.preview_html_path = html_path - self._log_action(f"Preview HTML written to {html_path}") - self._set_status("Preview generated", progress=self._weighted_progress("preview", 1)) - else: - self.preview_json_path = None - self.preview_html_path = None - self._set_status("Plan ready", progress=100) + tracks = self._gather_tracks(path) + if not tracks: + messagebox.showwarning("No Tracks", "No audio tracks were found in the selected library.") + self._set_status("Idle", progress=0) + return + self._set_fingerprint_status("Grouping duplicates…") + self._set_status("Grouping…", progress=self._weighted_progress("grouping", 0)) + cfg = load_config() + threshold_settings = self._current_threshold_settings() + exact_threshold = threshold_settings["exact_duplicate_threshold"] + near_threshold = threshold_settings["near_duplicate_threshold"] + mixed_codec_boost = threshold_settings["mixed_codec_threshold_boost"] + fingerprint_settings = self._current_fingerprint_settings() + self._log_action( + "Duplicate scan thresholds: " + f"exact={exact_threshold:.3f}, near={near_threshold:.3f}, mixed_codec_boost={mixed_codec_boost:.3f}" + ) + self._log_action(f"Fingerprint settings: {fingerprint_settings}") + plan = build_consolidation_plan( + tracks, + exact_duplicate_threshold=exact_threshold, + near_duplicate_threshold=near_threshold, + mixed_codec_threshold_boost=mixed_codec_boost, + fingerprint_settings=fingerprint_settings, + threshold_settings=threshold_settings, + ) + self._set_status("Grouping…", progress=self._weighted_progress("grouping", 1)) + self._plan = plan + self.group_disposition_overrides.clear() + self._apply_deletion_mode() - self._log_action( - f"Plan: {len(plan.groups)} groups, review required={plan.review_required_count}" + docs_dir = os.path.join(path, "Docs") + os.makedirs(docs_dir, exist_ok=True) + if write_preview: + self._set_fingerprint_status("Generating preview…") + self._set_status("Preview…", progress=self._weighted_progress("preview", 0)) + json_path = os.path.join(docs_dir, "duplicate_preview.json") + export_consolidation_preview(plan, json_path) + self.preview_json_path = json_path + self._log_action(f"Audit JSON written to {json_path}") + html_path = os.path.join(docs_dir, "duplicate_preview.html") + export_consolidation_preview_html( + plan, + html_path, + show_artwork_variants=self.show_artwork_variants_var.get(), ) - if plan.review_required_count: - self._log_action("Review required groups will block execution unless overridden.") - finally: - if timing_enabled: - plan_elapsed = time.monotonic() - plan_start_time - plan_end_ts = datetime.now().isoformat(timespec="seconds") - self._log_action( - f"Preview plan timing end: {plan_end_ts} (elapsed {plan_elapsed:.2f}s)" - ) + self.preview_html_path = html_path + self._log_action(f"Preview HTML written to {html_path}") + self._set_status("Preview generated", progress=self._weighted_progress("preview", 1)) + else: + self.preview_json_path = None + self.preview_html_path = None + self._set_status("Plan ready", progress=100) + + self._log_action( + f"Plan: {len(plan.groups)} groups, review required={plan.review_required_count}" + ) + if plan.review_required_count: + self._log_action("Review required groups will block execution unless overridden.") def _load_preview_plan(self, preview_path: str): try: @@ -2904,70 +2891,12 @@ def _handle_scan(self) -> None: self._log_action("Scan clicked") self._generate_plan(write_preview=False) - def _start_preview_heartbeat(self) -> None: - self._dup_preview_heartbeat_running = True - self._dup_preview_heartbeat_start = time.monotonic() - self._dup_preview_heartbeat_last_tick = None - self._dup_preview_heartbeat_count = 0 - self._log_action("Preview heartbeat started (100ms interval).") - self.after(int(self._dup_preview_heartbeat_interval * 1000), self._preview_heartbeat_tick) - - def _preview_heartbeat_tick(self) -> None: - if not self._dup_preview_heartbeat_running: - return - now = time.monotonic() - if self._dup_preview_heartbeat_last_tick is not None: - gap = now - self._dup_preview_heartbeat_last_tick - if gap > self._dup_preview_heartbeat_interval * 3: - self._log_action( - f"Preview heartbeat delayed by {gap:.2f}s while preview running." - ) - self._dup_preview_heartbeat_last_tick = now - self._dup_preview_heartbeat_count += 1 - self.after(int(self._dup_preview_heartbeat_interval * 1000), self._preview_heartbeat_tick) - - def _stop_preview_heartbeat(self) -> None: - if not self._dup_preview_heartbeat_running: - return - self._dup_preview_heartbeat_running = False - now = time.monotonic() - elapsed = 0.0 - if self._dup_preview_heartbeat_start is not None: - elapsed = now - self._dup_preview_heartbeat_start - if self._dup_preview_heartbeat_count == 0: - self._log_action( - f"Preview heartbeat did not fire during preview window ({elapsed:.2f}s)." - ) - elif ( - self._dup_preview_heartbeat_last_tick is not None - and now - self._dup_preview_heartbeat_last_tick - > self._dup_preview_heartbeat_interval * 3 - ): - stall = now - self._dup_preview_heartbeat_last_tick - self._log_action( - f"Preview heartbeat stalled for {stall:.2f}s before completion." - ) - self._log_action( - f"Preview heartbeat stopped after {elapsed:.2f}s " - f"({self._dup_preview_heartbeat_count} ticks)." - ) - def _handle_preview(self) -> None: self._reset_execute_confirmation() self._log_action("Preview clicked") - start_ts = datetime.now().isoformat(timespec="seconds") - start_time = time.monotonic() - self._log_action(f"Preview timing start: {start_ts}") - self._start_preview_heartbeat() - try: - self._generate_plan(write_preview=True) - if self.preview_html_path: - self._open_preview() - finally: - elapsed = time.monotonic() - start_time - end_ts = datetime.now().isoformat(timespec="seconds") - self._stop_preview_heartbeat() - self._log_action(f"Preview timing end: {end_ts} (elapsed {elapsed:.2f}s)") + self._generate_plan(write_preview=True) + if self.preview_html_path: + self._open_preview() def _handle_execute(self) -> None: if not self._execute_confirmation_pending: @@ -4317,11 +4246,6 @@ def __init__(self): self._preview_in_progress = False self._player_busy_item: str | None = None self._ignore_next_preview_finish = False - self._dup_preview_heartbeat_running = False - self._dup_preview_heartbeat_start: float | None = None - self._dup_preview_heartbeat_last_tick: float | None = None - self._dup_preview_heartbeat_count = 0 - self._dup_preview_heartbeat_interval = 0.1 # Player tab state self.player_tracks: list[dict[str, str]] = [] From 23cec89ac77c7fb31b7f1372116bdf55287ef2ae Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 6 Jan 2026 20:51:25 -0500 Subject: [PATCH 304/606] Reduce preview heartbeat logging overhead --- main_gui.py | 217 +++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 154 insertions(+), 63 deletions(-) diff --git a/main_gui.py b/main_gui.py index 444673c..adf1637 100644 --- a/main_gui.py +++ b/main_gui.py @@ -2803,69 +2803,84 @@ def _track_payload( return tracks def _generate_plan(self, write_preview: bool) -> None: - path = self._validate_library_root() - if not path: - return - - tracks = self._gather_tracks(path) - if not tracks: - messagebox.showwarning("No Tracks", "No audio tracks were found in the selected library.") - self._set_status("Idle", progress=0) - return - self._set_fingerprint_status("Grouping duplicates…") - self._set_status("Grouping…", progress=self._weighted_progress("grouping", 0)) - cfg = load_config() - threshold_settings = self._current_threshold_settings() - exact_threshold = threshold_settings["exact_duplicate_threshold"] - near_threshold = threshold_settings["near_duplicate_threshold"] - mixed_codec_boost = threshold_settings["mixed_codec_threshold_boost"] - fingerprint_settings = self._current_fingerprint_settings() - self._log_action( - "Duplicate scan thresholds: " - f"exact={exact_threshold:.3f}, near={near_threshold:.3f}, mixed_codec_boost={mixed_codec_boost:.3f}" - ) - self._log_action(f"Fingerprint settings: {fingerprint_settings}") - plan = build_consolidation_plan( - tracks, - exact_duplicate_threshold=exact_threshold, - near_duplicate_threshold=near_threshold, - mixed_codec_threshold_boost=mixed_codec_boost, - fingerprint_settings=fingerprint_settings, - threshold_settings=threshold_settings, - ) - self._set_status("Grouping…", progress=self._weighted_progress("grouping", 1)) - self._plan = plan - self.group_disposition_overrides.clear() - self._apply_deletion_mode() + timing_enabled = write_preview + if timing_enabled: + plan_start_ts = datetime.now().isoformat(timespec="seconds") + plan_start_time = time.monotonic() + logger.info("Preview plan timing start: %s", plan_start_ts) + try: + path = self._validate_library_root() + if not path: + return - docs_dir = os.path.join(path, "Docs") - os.makedirs(docs_dir, exist_ok=True) - if write_preview: - self._set_fingerprint_status("Generating preview…") - self._set_status("Preview…", progress=self._weighted_progress("preview", 0)) - json_path = os.path.join(docs_dir, "duplicate_preview.json") - export_consolidation_preview(plan, json_path) - self.preview_json_path = json_path - self._log_action(f"Audit JSON written to {json_path}") - html_path = os.path.join(docs_dir, "duplicate_preview.html") - export_consolidation_preview_html( - plan, - html_path, - show_artwork_variants=self.show_artwork_variants_var.get(), + tracks = self._gather_tracks(path) + if not tracks: + messagebox.showwarning("No Tracks", "No audio tracks were found in the selected library.") + self._set_status("Idle", progress=0) + return + self._set_fingerprint_status("Grouping duplicates…") + self._set_status("Grouping…", progress=self._weighted_progress("grouping", 0)) + cfg = load_config() + threshold_settings = self._current_threshold_settings() + exact_threshold = threshold_settings["exact_duplicate_threshold"] + near_threshold = threshold_settings["near_duplicate_threshold"] + mixed_codec_boost = threshold_settings["mixed_codec_threshold_boost"] + fingerprint_settings = self._current_fingerprint_settings() + self._log_action( + "Duplicate scan thresholds: " + f"exact={exact_threshold:.3f}, near={near_threshold:.3f}, mixed_codec_boost={mixed_codec_boost:.3f}" ) - self.preview_html_path = html_path - self._log_action(f"Preview HTML written to {html_path}") - self._set_status("Preview generated", progress=self._weighted_progress("preview", 1)) - else: - self.preview_json_path = None - self.preview_html_path = None - self._set_status("Plan ready", progress=100) + self._log_action(f"Fingerprint settings: {fingerprint_settings}") + plan = build_consolidation_plan( + tracks, + exact_duplicate_threshold=exact_threshold, + near_duplicate_threshold=near_threshold, + mixed_codec_threshold_boost=mixed_codec_boost, + fingerprint_settings=fingerprint_settings, + threshold_settings=threshold_settings, + ) + self._set_status("Grouping…", progress=self._weighted_progress("grouping", 1)) + self._plan = plan + self.group_disposition_overrides.clear() + self._apply_deletion_mode() - self._log_action( - f"Plan: {len(plan.groups)} groups, review required={plan.review_required_count}" - ) - if plan.review_required_count: - self._log_action("Review required groups will block execution unless overridden.") + docs_dir = os.path.join(path, "Docs") + os.makedirs(docs_dir, exist_ok=True) + if write_preview: + self._set_fingerprint_status("Generating preview…") + self._set_status("Preview…", progress=self._weighted_progress("preview", 0)) + json_path = os.path.join(docs_dir, "duplicate_preview.json") + export_consolidation_preview(plan, json_path) + self.preview_json_path = json_path + self._log_action(f"Audit JSON written to {json_path}") + html_path = os.path.join(docs_dir, "duplicate_preview.html") + export_consolidation_preview_html( + plan, + html_path, + show_artwork_variants=self.show_artwork_variants_var.get(), + ) + self.preview_html_path = html_path + self._log_action(f"Preview HTML written to {html_path}") + self._set_status("Preview generated", progress=self._weighted_progress("preview", 1)) + else: + self.preview_json_path = None + self.preview_html_path = None + self._set_status("Plan ready", progress=100) + + self._log_action( + f"Plan: {len(plan.groups)} groups, review required={plan.review_required_count}" + ) + if plan.review_required_count: + self._log_action("Review required groups will block execution unless overridden.") + finally: + if timing_enabled: + plan_elapsed = time.monotonic() - plan_start_time + plan_end_ts = datetime.now().isoformat(timespec="seconds") + logger.info( + "Preview plan timing end: %s (elapsed %.2fs)", + plan_end_ts, + plan_elapsed, + ) def _load_preview_plan(self, preview_path: str): try: @@ -2891,12 +2906,82 @@ def _handle_scan(self) -> None: self._log_action("Scan clicked") self._generate_plan(write_preview=False) + def _start_preview_heartbeat(self) -> None: + self._dup_preview_heartbeat_running = True + self._dup_preview_heartbeat_start = time.monotonic() + self._dup_preview_heartbeat_last_tick = None + self._dup_preview_heartbeat_count = 0 + self._dup_preview_heartbeat_stall_logged = False + self.after(int(self._dup_preview_heartbeat_interval * 1000), self._preview_heartbeat_tick) + + def _preview_heartbeat_tick(self) -> None: + if not self._dup_preview_heartbeat_running: + return + now = time.monotonic() + if self._dup_preview_heartbeat_last_tick is not None: + gap = now - self._dup_preview_heartbeat_last_tick + if ( + gap > self._dup_preview_heartbeat_interval * 10 + and not self._dup_preview_heartbeat_stall_logged + ): + logger.warning( + "Preview heartbeat delayed by %.2fs while preview running.", + gap, + ) + self._dup_preview_heartbeat_stall_logged = True + self._dup_preview_heartbeat_last_tick = now + self._dup_preview_heartbeat_count += 1 + self.after(int(self._dup_preview_heartbeat_interval * 1000), self._preview_heartbeat_tick) + + def _stop_preview_heartbeat(self) -> None: + if not self._dup_preview_heartbeat_running: + return + self._dup_preview_heartbeat_running = False + now = time.monotonic() + elapsed = 0.0 + if self._dup_preview_heartbeat_start is not None: + elapsed = now - self._dup_preview_heartbeat_start + if self._dup_preview_heartbeat_count == 0: + logger.warning( + "Preview heartbeat did not fire during preview window (%.2fs).", + elapsed, + ) + elif ( + self._dup_preview_heartbeat_last_tick is not None + and now - self._dup_preview_heartbeat_last_tick + > self._dup_preview_heartbeat_interval * 10 + ): + stall = now - self._dup_preview_heartbeat_last_tick + logger.warning( + "Preview heartbeat stalled for %.2fs before completion.", + stall, + ) + logger.info( + "Preview heartbeat stopped after %.2fs (%s ticks).", + elapsed, + self._dup_preview_heartbeat_count, + ) + def _handle_preview(self) -> None: self._reset_execute_confirmation() self._log_action("Preview clicked") - self._generate_plan(write_preview=True) - if self.preview_html_path: - self._open_preview() + start_ts = datetime.now().isoformat(timespec="seconds") + start_time = time.monotonic() + logger.info("Preview timing start: %s", start_ts) + self._start_preview_heartbeat() + try: + self._generate_plan(write_preview=True) + if self.preview_html_path: + self._open_preview() + finally: + elapsed = time.monotonic() - start_time + end_ts = datetime.now().isoformat(timespec="seconds") + self._stop_preview_heartbeat() + logger.info( + "Preview timing end: %s (elapsed %.2fs)", + end_ts, + elapsed, + ) def _handle_execute(self) -> None: if not self._execute_confirmation_pending: @@ -4246,6 +4331,12 @@ def __init__(self): self._preview_in_progress = False self._player_busy_item: str | None = None self._ignore_next_preview_finish = False + self._dup_preview_heartbeat_running = False + self._dup_preview_heartbeat_start: float | None = None + self._dup_preview_heartbeat_last_tick: float | None = None + self._dup_preview_heartbeat_count = 0 + self._dup_preview_heartbeat_interval = 0.1 + self._dup_preview_heartbeat_stall_logged = False # Player tab state self.player_tracks: list[dict[str, str]] = [] From 65f1d7a33b6b78ffb8c59e4884eece2ae59b9cda Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 6 Jan 2026 20:54:23 -0500 Subject: [PATCH 305/606] Add no-op group toggle to duplicate finder --- duplicate_consolidation.py | 118 +++++++++++++++++++------------------ main_gui.py | 24 ++++++++ 2 files changed, 85 insertions(+), 57 deletions(-) diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index f33bda0..d979198 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -1003,6 +1003,64 @@ def refresh_plan_signature(self) -> str: return signature +def _planned_actions(group: GroupPlan) -> List[Dict[str, object]]: + actions: List[Dict[str, object]] = [] + for path, changes in group.metadata_changes.items(): + fields = sorted(changes.keys()) if isinstance(changes, Mapping) else list(changes or []) + actions.append( + { + "step": "metadata", + "target": path, + "status": "review", + "detail": "Planned metadata updates.", + "metadata": {"fields": fields}, + } + ) + for directive in group.artwork: + actions.append( + { + "step": "artwork", + "target": directive.target, + "status": "review", + "detail": f"Migrate artwork from {directive.source} to {directive.target}.", + "metadata": {"source": directive.source, "reason": directive.reason}, + } + ) + for loser in group.losers: + disposition = group.loser_disposition.get(loser, "quarantine") + actions.append( + { + "step": "loser_cleanup", + "target": loser, + "status": "review", + "detail": f"Loser marked for {disposition}.", + "metadata": {"disposition": disposition}, + } + ) + for playlist, destination in group.playlist_rewrites.items(): + actions.append( + { + "step": "playlist", + "target": playlist, + "status": "review", + "detail": "Planned playlist rewrite.", + "metadata": {"destination": destination}, + } + ) + return actions + + +def _is_noop_group(actions: List[Dict[str, object]]) -> bool: + has_non_loser_action = any( + act["step"] in {"metadata", "artwork", "playlist"} for act in actions + ) + has_non_retain_loser = any( + act["step"] == "loser_cleanup" and act.get("metadata", {}).get("disposition") != "retain" + for act in actions + ) + return not has_non_loser_action and not has_non_retain_loser + + @dataclass class PairInspectionStep: name: str @@ -2892,62 +2950,6 @@ def _format_notes(detail: str, meta_notes: str) -> str: return f"{detail_html}
      {meta_html}
      " return detail_html - def _planned_actions(group: GroupPlan) -> List[Dict[str, object]]: - actions: List[Dict[str, object]] = [] - for path, changes in group.metadata_changes.items(): - fields = sorted(changes.keys()) - actions.append( - { - "step": "metadata", - "target": path, - "status": "review", - "detail": "Planned metadata updates.", - "metadata": {"fields": fields}, - } - ) - for directive in group.artwork: - actions.append( - { - "step": "artwork", - "target": directive.target, - "status": "review", - "detail": f"Migrate artwork from {directive.source} to {directive.target}.", - "metadata": {"source": directive.source, "reason": directive.reason}, - } - ) - for loser in group.losers: - disposition = group.loser_disposition.get(loser, "quarantine") - actions.append( - { - "step": "loser_cleanup", - "target": loser, - "status": "review", - "detail": f"Loser marked for {disposition}.", - "metadata": {"disposition": disposition}, - } - ) - for playlist, destination in group.playlist_rewrites.items(): - actions.append( - { - "step": "playlist", - "target": playlist, - "status": "review", - "detail": "Planned playlist rewrite.", - "metadata": {"destination": destination}, - } - ) - return actions - - def _is_noop_group(actions: List[Dict[str, object]]) -> bool: - has_non_loser_action = any( - act["step"] in {"metadata", "artwork", "playlist"} for act in actions - ) - has_non_retain_loser = any( - act["step"] == "loser_cleanup" and act.get("metadata", {}).get("disposition") != "retain" - for act in actions - ) - return not has_non_loser_action and not has_non_retain_loser - def _missing_artwork_gate(group: GroupPlan) -> bool: for note in group.artwork_evidence: lowered = note.lower() @@ -3748,7 +3750,9 @@ def _format_setting(value: object) -> str: "", "", "", - "", + "", + "Hides groups with no planned operations when off.", "
      ", "
      ", "0 visible", diff --git a/main_gui.py b/main_gui.py index 444673c..2b570e8 100644 --- a/main_gui.py +++ b/main_gui.py @@ -46,6 +46,8 @@ get_tags, ) from duplicate_consolidation import ( + _is_noop_group, + _planned_actions, build_consolidation_plan, build_duplicate_pair_report, consolidation_plan_from_dict, @@ -1976,6 +1978,7 @@ def __init__(self, parent: tk.Widget, library_path: str): self.show_artwork_variants_var = tk.BooleanVar( value=cfg.get("duplicate_finder_show_artwork_variants", True) ) + self.show_noop_groups_var = tk.BooleanVar(value=False) self.group_disposition_var = tk.StringVar(value="") self.group_disposition_overrides: dict[str, str] = {} self._selected_group_id: str | None = None @@ -2061,6 +2064,17 @@ def __init__(self, parent: tk.Widget, library_path: str): variable=self.show_artwork_variants_var, command=self._toggle_show_artwork_variants, ).pack(side="left", padx=(0, 10)) + show_noop_toggle = ttk.Checkbutton( + controls, + text="Show no-op groups", + variable=self.show_noop_groups_var, + command=self._toggle_show_noop_groups, + ) + show_noop_toggle.pack(side="left", padx=(0, 10)) + Tooltip( + show_noop_toggle, + lambda: "When unchecked, hides groups with no planned operations.", + ) ttk.Checkbutton( controls, text="Update Playlists", @@ -2265,7 +2279,11 @@ def _reset_group_selection(self) -> None: def _update_groups_view(self, plan) -> None: self.groups_tree.delete(*self.groups_tree.get_children()) + show_noop_groups = self.show_noop_groups_var.get() for group in plan.groups: + actions = _planned_actions(group) + if _is_noop_group(actions) and not show_noop_groups: + continue title = plan and group.planned_winner_tags.get("title") if hasattr(group, "planned_winner_tags") else None status = "Review" if group.review_flags else "Ready" self.groups_tree.insert( @@ -2282,6 +2300,12 @@ def _update_groups_view(self, plan) -> None: self.group_details.configure(state="disabled") self._reset_group_selection() + def _toggle_show_noop_groups(self) -> None: + state = "enabled" if self.show_noop_groups_var.get() else "disabled" + self._log_action(f"Show no-op groups {state}") + if self._plan: + self._update_groups_view(self._plan) + def _on_group_select(self, event=None) -> None: sel = self.groups_tree.selection() if not sel or not self._plan: From afef1b2e3beb484fa70c7ba91667751b7b240580 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 6 Jan 2026 20:57:56 -0500 Subject: [PATCH 306/606] Fix duplicate finder heartbeat and finish handling --- main_gui.py | 124 +++++++++++++++++++++++++++++----------------------- 1 file changed, 70 insertions(+), 54 deletions(-) diff --git a/main_gui.py b/main_gui.py index adf1637..c7e0260 100644 --- a/main_gui.py +++ b/main_gui.py @@ -1988,6 +1988,12 @@ def __init__(self, parent: tk.Widget, library_path: str): "grouping": 0.2, "preview": 0.1, } + self._dup_preview_heartbeat_running = False + self._dup_preview_heartbeat_start: float | None = None + self._dup_preview_heartbeat_last_tick: float | None = None + self._dup_preview_heartbeat_count = 0 + self._dup_preview_heartbeat_interval = 0.1 + self._dup_preview_heartbeat_stall_logged = False container = ttk.Frame(self) container.pack(fill="both", expand=True, padx=10, pady=10) @@ -3136,63 +3142,73 @@ def log_callback(msg: str) -> None: self._reset_execute_confirmation() def finish(result, error: Exception | None = None) -> None: - if error is not None: - self._log_action(f"Execution failed: {error}") - self._set_status("Execution failed", progress=100) - messagebox.showerror("Execution Failed", str(error)) - self._reset_execute_confirmation() - return - cancelled = any(action.status == "cancelled" for action in result.actions) - if cancelled: - status = "Execution cancelled" - else: - status = "Executed" if result.success else "Execution failed" - self._set_status(status, progress=100) - if cancelled: - self._log_action("Execution complete: cancelled") - else: - self._log_action(f"Execution complete: {'success' if result.success else 'failed'}") - report_path = result.report_paths.get("html_report") - self._log_action(f"Execution report: {report_path}") - if report_path: - if not os.path.exists(ensure_long_path(report_path)): - self._log_action("Execution report missing at expected path; enabling Open Report anyway.") - self.execution_report_path = report_path - self.open_report_btn.config(state="normal") - else: - self.execution_report_path = None - self.open_report_btn.config(state="disabled") - if cancelled: - report_line = "" - if report_path: - if os.path.exists(ensure_long_path(report_path)): - report_line = f"\n\nReport (HTML): {report_path}" - else: - report_line = ( - "\n\nReport (HTML) was not found at the expected path:\n" - f"{report_path}" - ) - messagebox.showinfo( - "Execution Cancelled", - "Execution was cancelled. Review the report for details." - f"{report_line}", - ) - elif not result.success: - report_line = "" + try: + if error is not None: + self._log_action(f"Execution failed: {error}") + self._set_status("Execution failed", progress=100) + messagebox.showerror("Execution Failed", str(error)) + self._reset_execute_confirmation() + return + cancelled = any(action.status == "cancelled" for action in result.actions) + if cancelled: + status = "Execution cancelled" + else: + status = "Executed" if result.success else "Execution failed" + self._set_status(status, progress=100) + if cancelled: + self._log_action("Execution complete: cancelled") + else: + self._log_action(f"Execution complete: {'success' if result.success else 'failed'}") + report_path = result.report_paths.get("html_report") + self._log_action(f"Execution report: {report_path}") if report_path: - if os.path.exists(ensure_long_path(report_path)): - report_line = f"\n\nReport (HTML): {report_path}" - else: - report_line = ( - "\n\nReport (HTML) was not found at the expected path:\n" - f"{report_path}" - ) - messagebox.showwarning( + if not os.path.exists(ensure_long_path(report_path)): + self._log_action("Execution report missing at expected path; enabling Open Report anyway.") + self.execution_report_path = report_path + self.open_report_btn.config(state="normal") + else: + self.execution_report_path = None + self.open_report_btn.config(state="disabled") + if cancelled: + report_line = "" + if report_path: + if os.path.exists(ensure_long_path(report_path)): + report_line = f"\n\nReport (HTML): {report_path}" + else: + report_line = ( + "\n\nReport (HTML) was not found at the expected path:\n" + f"{report_path}" + ) + messagebox.showinfo( + "Execution Cancelled", + "Execution was cancelled. Review the report for details." + f"{report_line}", + ) + elif not result.success: + report_line = "" + if report_path: + if os.path.exists(ensure_long_path(report_path)): + report_line = f"\n\nReport (HTML): {report_path}" + else: + report_line = ( + "\n\nReport (HTML) was not found at the expected path:\n" + f"{report_path}" + ) + messagebox.showwarning( + "Execution Failed", + "Execution completed with errors. Review the report for details." + f"{report_line}", + ) + self._reset_execute_confirmation() + except Exception as exc: + self._log_action(f"Execution finish handler failed: {exc}") + self._set_status("Execution failed", progress=100) + messagebox.showerror( "Execution Failed", - "Execution completed with errors. Review the report for details." - f"{report_line}", + "An unexpected error occurred while finalizing execution. " + "Check the log for details.", ) - self._reset_execute_confirmation() + self._reset_execute_confirmation() def worker() -> None: try: From a6a1e0b2043a862276ca3ba1cc9fbf98f64701a8 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 6 Jan 2026 21:58:35 -0500 Subject: [PATCH 307/606] Refactor preview generation to background thread --- main_gui.py | 284 +++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 212 insertions(+), 72 deletions(-) diff --git a/main_gui.py b/main_gui.py index 0fb605b..62f701b 100644 --- a/main_gui.py +++ b/main_gui.py @@ -38,6 +38,7 @@ import time from datetime import datetime from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import dataclass from validator import validate_soundvault_structure from music_indexer_api import ( @@ -125,6 +126,17 @@ ) from utils.opus_metadata_reader import read_opus_metadata + +@dataclass +class PlanGenerationResult: + plan: Any | None + preview_json_path: str | None + preview_html_path: str | None + log_messages: list[str] + review_required_count: int + group_count: int + had_tracks: bool + FilterFn = Callable[[FileRecord], bool] _cached_filters = None @@ -2689,9 +2701,17 @@ def _save() -> None: ttk.Button(btn_frame, text="OK", command=_save).pack() dlg.wait_window() - def _gather_tracks(self, library_root: str) -> list[dict[str, object]]: + def _gather_tracks( + self, + library_root: str, + *, + log_callback: Callable[[str], None] | None = None, + status_callback: Callable[[str, float], None] | None = None, + fingerprint_status_callback: Callable[[str], None] | None = None, + idle_callback: Callable[[], None] | None = None, + ) -> tuple[list[dict[str, object]], int, int]: if not library_root: - return [] + return [], 0, 0 docs_dir = os.path.join(library_root, "Docs") os.makedirs(docs_dir, exist_ok=True) db_path = os.path.join(docs_dir, ".duplicate_fingerprints.db") @@ -2711,12 +2731,17 @@ def _gather_tracks(self, library_root: str) -> list[dict[str, object]]: tracks: list[dict[str, object]] = [] total = len(audio_paths) if total == 0: - self._set_fingerprint_status("No eligible audio files found.") - self._set_status("Idle", progress=0) - return tracks - self._set_fingerprint_status(f"Fingerprinting 0/{total}") - self._set_status("Fingerprinting…", progress=self._weighted_progress("fingerprinting", 0)) - self.update_idletasks() + if fingerprint_status_callback: + fingerprint_status_callback("No eligible audio files found.") + if status_callback: + status_callback("Idle", progress=0) + return tracks, 0, 0 + if fingerprint_status_callback: + fingerprint_status_callback(f"Fingerprinting 0/{total}") + if status_callback: + status_callback("Fingerprinting…", progress=self._weighted_progress("fingerprinting", 0)) + if idle_callback: + idle_callback() last_update = time.monotonic() update_interval = 0.2 def _fingerprint_worker(target_path: str) -> tuple[str, int | None, str | None, str | None]: @@ -2767,7 +2792,7 @@ def _track_payload( sorted_paths = sorted(audio_paths) for path in sorted_paths: fingerprint_trace: dict[str, object] = {} - fp = get_cached_fingerprint(path, db_path, log_callback=self._log_action, trace=fingerprint_trace) + fp = get_cached_fingerprint(path, db_path, log_callback=log_callback, trace=fingerprint_trace) if fp: tracks_map[path] = _track_payload(path, fp, fingerprint_trace) completed += 1 @@ -2780,12 +2805,15 @@ def _track_payload( pending_paths.append(path) now = time.monotonic() if completed == total or now - last_update >= update_interval: - self._set_fingerprint_status(f"Fingerprinting {completed}/{total}") - self._set_status( - "Fingerprinting…", - progress=self._weighted_progress("fingerprinting", completed / total), - ) - self.update_idletasks() + if fingerprint_status_callback: + fingerprint_status_callback(f"Fingerprinting {completed}/{total}") + if status_callback: + status_callback( + "Fingerprinting…", + progress=self._weighted_progress("fingerprinting", completed / total), + ) + if idle_callback: + idle_callback() last_update = now max_workers = min(8, os.cpu_count() or 4) @@ -2804,63 +2832,100 @@ def _track_payload( completed += 1 now = time.monotonic() if completed == total or now - last_update >= update_interval: - self._set_fingerprint_status(f"Fingerprinting {completed}/{total}") - self._set_status( - "Fingerprinting…", - progress=self._weighted_progress("fingerprinting", completed / total), - ) - self.update_idletasks() + if fingerprint_status_callback: + fingerprint_status_callback(f"Fingerprinting {completed}/{total}") + if status_callback: + status_callback( + "Fingerprinting…", + progress=self._weighted_progress("fingerprinting", completed / total), + ) + if idle_callback: + idle_callback() last_update = now if error: failure_count += 1 - self._log_action(f"! Fingerprint failed for {path}: {error}") + if log_callback: + log_callback(f"! Fingerprint failed for {path}: {error}") continue if not fp: missing_count += 1 - self._log_action(f"! Missing fingerprint for {path}") + if log_callback: + log_callback(f"! Missing fingerprint for {path}") continue - if not store_fingerprint(path, db_path, duration, fp, log_callback=self._log_action): + if not store_fingerprint(path, db_path, duration, fp, log_callback=log_callback): failure_count += 1 fingerprint_trace = {"source": "computed", "error": ""} tracks_map[path] = _track_payload(path, fp, fingerprint_trace) - if missing_count or failure_count: - self._log_action( - f"Fingerprinting complete with {missing_count} missing fingerprints and " - f"{failure_count} failures." - ) tracks = [tracks_map[path] for path in sorted_paths if path in tracks_map] - return tracks + return tracks, missing_count, failure_count - def _generate_plan(self, write_preview: bool) -> None: + def _generate_plan( + self, + library_root: str, + *, + write_preview: bool, + threshold_settings: Mapping[str, float], + fingerprint_settings: Mapping[str, object], + show_artwork_variants: bool, + log_callback: Callable[[str], None] | None = None, + status_callback: Callable[[str, float], None] | None = None, + fingerprint_status_callback: Callable[[str], None] | None = None, + idle_callback: Callable[[], None] | None = None, + ) -> PlanGenerationResult: timing_enabled = write_preview + log_messages: list[str] = [] + + def log(msg: str) -> None: + log_messages.append(msg) + if log_callback: + log_callback(msg) + + def report_status(label: str, progress: float) -> None: + if status_callback: + status_callback(label, progress=progress) + + def report_fingerprint_status(message: str) -> None: + if fingerprint_status_callback: + fingerprint_status_callback(message) + if timing_enabled: plan_start_ts = datetime.now().isoformat(timespec="seconds") plan_start_time = time.monotonic() logger.info("Preview plan timing start: %s", plan_start_ts) try: - path = self._validate_library_root() - if not path: - return - - tracks = self._gather_tracks(path) + tracks, missing_count, failure_count = self._gather_tracks( + library_root, + log_callback=log, + status_callback=status_callback, + fingerprint_status_callback=fingerprint_status_callback, + idle_callback=idle_callback, + ) if not tracks: - messagebox.showwarning("No Tracks", "No audio tracks were found in the selected library.") - self._set_status("Idle", progress=0) - return - self._set_fingerprint_status("Grouping duplicates…") - self._set_status("Grouping…", progress=self._weighted_progress("grouping", 0)) - cfg = load_config() - threshold_settings = self._current_threshold_settings() + return PlanGenerationResult( + plan=None, + preview_json_path=None, + preview_html_path=None, + log_messages=log_messages, + review_required_count=0, + group_count=0, + had_tracks=False, + ) + if missing_count or failure_count: + log( + f"Fingerprinting complete with {missing_count} missing fingerprints and " + f"{failure_count} failures." + ) + report_fingerprint_status("Grouping duplicates…") + report_status("Grouping…", progress=self._weighted_progress("grouping", 0)) exact_threshold = threshold_settings["exact_duplicate_threshold"] near_threshold = threshold_settings["near_duplicate_threshold"] mixed_codec_boost = threshold_settings["mixed_codec_threshold_boost"] - fingerprint_settings = self._current_fingerprint_settings() - self._log_action( + log( "Duplicate scan thresholds: " f"exact={exact_threshold:.3f}, near={near_threshold:.3f}, mixed_codec_boost={mixed_codec_boost:.3f}" ) - self._log_action(f"Fingerprint settings: {fingerprint_settings}") + log(f"Fingerprint settings: {fingerprint_settings}") plan = build_consolidation_plan( tracks, exact_duplicate_threshold=exact_threshold, @@ -2869,39 +2934,43 @@ def _generate_plan(self, write_preview: bool) -> None: fingerprint_settings=fingerprint_settings, threshold_settings=threshold_settings, ) - self._set_status("Grouping…", progress=self._weighted_progress("grouping", 1)) - self._plan = plan - self.group_disposition_overrides.clear() - self._apply_deletion_mode() + report_status("Grouping…", progress=self._weighted_progress("grouping", 1)) - docs_dir = os.path.join(path, "Docs") + docs_dir = os.path.join(library_root, "Docs") os.makedirs(docs_dir, exist_ok=True) + preview_json_path = None + preview_html_path = None if write_preview: - self._set_fingerprint_status("Generating preview…") - self._set_status("Preview…", progress=self._weighted_progress("preview", 0)) + report_fingerprint_status("Generating preview…") + report_status("Preview…", progress=self._weighted_progress("preview", 0)) json_path = os.path.join(docs_dir, "duplicate_preview.json") export_consolidation_preview(plan, json_path) - self.preview_json_path = json_path - self._log_action(f"Audit JSON written to {json_path}") + preview_json_path = json_path + log(f"Audit JSON written to {json_path}") html_path = os.path.join(docs_dir, "duplicate_preview.html") export_consolidation_preview_html( plan, html_path, - show_artwork_variants=self.show_artwork_variants_var.get(), + show_artwork_variants=show_artwork_variants, ) - self.preview_html_path = html_path - self._log_action(f"Preview HTML written to {html_path}") - self._set_status("Preview generated", progress=self._weighted_progress("preview", 1)) + preview_html_path = html_path + log(f"Preview HTML written to {html_path}") + report_status("Preview generated", progress=self._weighted_progress("preview", 1)) else: - self.preview_json_path = None - self.preview_html_path = None - self._set_status("Plan ready", progress=100) + report_status("Plan ready", progress=100) - self._log_action( - f"Plan: {len(plan.groups)} groups, review required={plan.review_required_count}" - ) + log(f"Plan: {len(plan.groups)} groups, review required={plan.review_required_count}") if plan.review_required_count: - self._log_action("Review required groups will block execution unless overridden.") + log("Review required groups will block execution unless overridden.") + return PlanGenerationResult( + plan=plan, + preview_json_path=preview_json_path, + preview_html_path=preview_html_path, + log_messages=log_messages, + review_required_count=plan.review_required_count, + group_count=len(plan.groups), + had_tracks=True, + ) finally: if timing_enabled: plan_elapsed = time.monotonic() - plan_start_time @@ -2934,7 +3003,27 @@ def _handle_scan(self) -> None: return self._reset_execute_confirmation() self._log_action("Scan clicked") - self._generate_plan(write_preview=False) + threshold_settings = self._current_threshold_settings() + fingerprint_settings = self._current_fingerprint_settings() + result = self._generate_plan( + path, + write_preview=False, + threshold_settings=threshold_settings, + fingerprint_settings=fingerprint_settings, + show_artwork_variants=self.show_artwork_variants_var.get(), + log_callback=self._log_action, + status_callback=self._set_status, + fingerprint_status_callback=self._set_fingerprint_status, + idle_callback=self.update_idletasks, + ) + if not result.had_tracks: + messagebox.showwarning("No Tracks", "No audio tracks were found in the selected library.") + return + self._plan = result.plan + self.group_disposition_overrides.clear() + self._apply_deletion_mode() + self.preview_json_path = None + self.preview_html_path = None def _start_preview_heartbeat(self) -> None: self._dup_preview_heartbeat_running = True @@ -2993,17 +3082,22 @@ def _stop_preview_heartbeat(self) -> None: ) def _handle_preview(self) -> None: + path = self._validate_library_root() + if not path: + return self._reset_execute_confirmation() self._log_action("Preview clicked") start_ts = datetime.now().isoformat(timespec="seconds") start_time = time.monotonic() logger.info("Preview timing start: %s", start_ts) self._start_preview_heartbeat() - try: - self._generate_plan(write_preview=True) - if self.preview_html_path: - self._open_preview() - finally: + self._set_fingerprint_status("Starting preview…") + self._set_status("Preview…", progress=0) + threshold_settings = self._current_threshold_settings() + fingerprint_settings = self._current_fingerprint_settings() + show_artwork_variants = self.show_artwork_variants_var.get() + + def finalize_preview() -> None: elapsed = time.monotonic() - start_time end_ts = datetime.now().isoformat(timespec="seconds") self._stop_preview_heartbeat() @@ -3013,6 +3107,52 @@ def _handle_preview(self) -> None: elapsed, ) + def finish(result: PlanGenerationResult | None, error: Exception | None = None) -> None: + def schedule(callable_obj, *args) -> None: + self.after(0, callable_obj, *args) + + if error is not None: + schedule(self._log_action, f"Preview failed: {error}") + schedule(self._set_status, "Preview failed", 100) + schedule(messagebox.showerror, "Preview Failed", str(error)) + schedule(finalize_preview) + return + + if result is None or not result.had_tracks: + schedule(messagebox.showwarning, "No Tracks", "No audio tracks were found in the selected library.") + schedule(self._set_status, "Idle", 0) + schedule(finalize_preview) + return + + self._plan = result.plan + self.group_disposition_overrides.clear() + self._apply_deletion_mode() + self.preview_json_path = result.preview_json_path + self.preview_html_path = result.preview_html_path + + for message in result.log_messages: + schedule(self._log_action, message) + schedule(self._set_status, "Preview generated", self._weighted_progress("preview", 1)) + if self.preview_html_path: + schedule(self._open_preview) + schedule(finalize_preview) + + def worker() -> None: + try: + result = self._generate_plan( + path, + write_preview=True, + threshold_settings=threshold_settings, + fingerprint_settings=fingerprint_settings, + show_artwork_variants=show_artwork_variants, + ) + except Exception as exc: + self.after(0, finish, None, exc) + return + self.after(0, finish, result, None) + + threading.Thread(target=worker, daemon=True).start() + def _handle_execute(self) -> None: if not self._execute_confirmation_pending: self._execute_confirmation_pending = True From 5c3b81fc167a776b61147e84f9ae8c0c3b1085c3 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 6 Jan 2026 22:02:28 -0500 Subject: [PATCH 308/606] Hide ready groups in preview by default --- duplicate_consolidation.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/duplicate_consolidation.py b/duplicate_consolidation.py index d979198..fd64725 100644 --- a/duplicate_consolidation.py +++ b/duplicate_consolidation.py @@ -3750,6 +3750,8 @@ def _format_setting(value: object) -> str: "", "", "", + "", "", "Hides groups with no planned operations when off.", @@ -3821,6 +3823,7 @@ def _format_setting(value: object) -> str: f"data-type='{esc(group_type_hint)}' " f"data-search='{esc(search_text.lower())}' " f"data-no-op='{str(is_noop).lower()}' " + f"data-status='{esc(state)}' " f"data-quarantine-count='{group_disposition_count}'>" ) album_art_src = _album_art_src(group, group.winner_path) @@ -4163,6 +4166,7 @@ def _format_setting(value: object) -> str: "const groups = () => $$('#groups details.group');" "const searchEl = $('#search');" "const filterEl = $('#filter');" + "const showReadyEl = $('#showReady');" "const showNoOpsEl = $('#showNoOps');" "const visibleCountEl = $('#visibleCount');" "const expandAllBtn = $('#expandAll');" @@ -4170,8 +4174,14 @@ def _format_setting(value: object) -> str: "const settingsBtn = $('#toggleSettings');" "const settingsPanel = $('#settingsPanel');" "function computeStats(){" + "const showReady = showReadyEl ? showReadyEl.checked : false;" "const showNoOps = showNoOpsEl ? showNoOpsEl.checked : false;" - "const g = groups().filter(d => showNoOps || (d.getAttribute('data-no-op') || 'false') !== 'true');" + "const g = groups().filter(d => {" + "const isNoOp = (d.getAttribute('data-no-op') || 'false') === 'true';" + "const status = (d.getAttribute('data-status') || '').toLowerCase();" + "const isReady = status === 'ready';" + "return (showNoOps || !isNoOp) && (showReady || !isReady);" + "});" "$('#statGroups').textContent = g.length.toString();" "$('#statWinners').textContent = g.length.toString();" "let qTotal = 0;" @@ -4201,6 +4211,7 @@ def _format_setting(value: object) -> str: "function applyFilters(){" "const term = (searchEl.value || '').trim().toLowerCase();" "const mode = filterEl.value;" + "const showReady = showReadyEl ? showReadyEl.checked : false;" "const showNoOps = showNoOpsEl ? showNoOpsEl.checked : false;" "let visible = 0;" "for (const d of groups()){" @@ -4210,11 +4221,13 @@ def _format_setting(value: object) -> str: "const hasFailure = (d.getAttribute('data-has-failure') || 'false') === 'true';" "const typeHint = (d.getAttribute('data-type') || '').toLowerCase();" "const isNoOp = (d.getAttribute('data-no-op') || 'false') === 'true';" + "const status = (d.getAttribute('data-status') || '').toLowerCase();" + "const isReady = status === 'ready';" "let matchesFilter = true;" "if (mode === 'has-quarantine') matchesFilter = hasQuarantine;" "if (mode === 'metadata-only') matchesFilter = typeHint === 'metadata-only';" "if (mode === 'failed') matchesFilter = hasFailure;" - "const show = matchesSearch && matchesFilter && (showNoOps || !isNoOp);" + "const show = matchesSearch && matchesFilter && (showNoOps || !isNoOp) && (showReady || !isReady);" "d.classList.toggle('hidden', !show);" "if (show) visible += 1;" "}" @@ -4272,6 +4285,7 @@ def _format_setting(value: object) -> str: "}" "searchEl?.addEventListener('input', applyFilters);" "filterEl?.addEventListener('change', applyFilters);" + "showReadyEl?.addEventListener('change', applyFilters);" "showNoOpsEl?.addEventListener('change', applyFilters);" "computeStats();" "applyFilters();" From bd88864e0d3fa43553fdbcaeebb6516888dc688b Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 6 Jan 2026 22:51:46 -0500 Subject: [PATCH 309/606] Add coarse gating to simple duplicate finder --- simple_duplicate_finder.py | 37 ++++++++++--- tests/test_simple_duplicate_finder.py | 79 +++++++++++++++++++++++++-- 2 files changed, 104 insertions(+), 12 deletions(-) diff --git a/simple_duplicate_finder.py b/simple_duplicate_finder.py index c146c30..0f5ef83 100644 --- a/simple_duplicate_finder.py +++ b/simple_duplicate_finder.py @@ -5,7 +5,7 @@ from typing import Callable, List, Tuple, Dict, Optional, Iterator from fingerprint_cache import get_fingerprint -from near_duplicate_detector import fingerprint_distance +from near_duplicate_detector import fingerprint_distance, _coarse_fingerprint_keys, _coarse_gate import chromaprint_utils from config import load_config, FP_DURATION_MS, FP_OFFSET_MS, FP_SILENCE_MIN_LEN_MS, FP_SILENCE_THRESHOLD_DB @@ -148,20 +148,39 @@ def compute(path: str) -> Tuple[Optional[int], Optional[str]]: groups: List[Dict[str, object]] = [] prefix_map: Dict[str, List[Dict[str, object]]] = {} + coarse_index: Dict[str, List[Dict[str, object]]] = {} use_prefix = prefix_len is not None and prefix_len > 0 def handle_fingerprint(path: str, fp: str) -> None: prefix = fp[:prefix_len] if use_prefix else "" log_callback(f"[GROUP] path={path}, prefix={prefix}") + coarse_keys = _coarse_fingerprint_keys(fp) + candidate_groups: Dict[int, Dict[str, object]] = {} + if coarse_keys: + for key in coarse_keys: + for g in coarse_index.get(key, []): + candidate_groups[id(g)] = g + else: + for groups_for_key in prefix_map.values(): + for g in groups_for_key: + candidate_groups[id(g)] = g + + coarse_candidate_count = len(candidate_groups) cand_groups: List[Tuple[Dict[str, object], str]] = [] if use_prefix: - for key, groups_for_key in prefix_map.items(): - if prefix_distance(prefix, key) <= PREFIX_THRESHOLD: - for g in groups_for_key: - cand_groups.append((g, key)) + for g in candidate_groups.values(): + group_prefix = g.get("prefix", "") + if not coarse_keys or _coarse_gate(coarse_keys, g.get("coarse_keys", [])): + if prefix_distance(prefix, group_prefix) <= PREFIX_THRESHOLD: + cand_groups.append((g, group_prefix)) else: - for g in prefix_map.get(prefix, []): - cand_groups.append((g, prefix)) + for g in candidate_groups.values(): + if _coarse_gate(coarse_keys, g.get("coarse_keys", [])) or not coarse_keys: + cand_groups.append((g, g.get("prefix", ""))) + log_callback( + f"[COARSE] {path} coarse candidates {coarse_candidate_count} -> " + f"prefix candidates {len(cand_groups)}" + ) _dlog("GROUP", f"file {path} -> prefix {prefix}") _dlog("GROUP", f"{len(cand_groups)} groups for prefix") placed = False @@ -181,9 +200,11 @@ def handle_fingerprint(path: str, fp: str) -> None: placed = True break if not placed: - g = {"fp": fp, "paths": [path]} + g = {"fp": fp, "paths": [path], "prefix": prefix, "coarse_keys": coarse_keys} prefix_map.setdefault(prefix, []).append(g) groups.append(g) + for key in coarse_keys: + coarse_index.setdefault(key, []).append(g) _dlog("GROUP", f"new group for prefix {prefix}") with ThreadPoolExecutor(max_workers=max_workers) as executor: diff --git a/tests/test_simple_duplicate_finder.py b/tests/test_simple_duplicate_finder.py index 64e64ef..d884461 100644 --- a/tests/test_simple_duplicate_finder.py +++ b/tests/test_simple_duplicate_finder.py @@ -4,7 +4,7 @@ import importlib import simple_duplicate_finder as sdf -from fingerprint_cache import flush_cache +from fingerprint_cache import flush_cache, _ensure_db def load_module(monkeypatch, fp_map=None): @@ -23,13 +23,55 @@ def load_module(monkeypatch, fp_map=None): return sdf +def _naive_duplicate_pairs(sdf_mod, ordered_paths, fp_map, prefix_len=None, threshold=0.03): + groups = [] + prefix_map = {} + use_prefix = prefix_len is not None and prefix_len > 0 + for path in ordered_paths: + fp = fp_map[os.path.basename(path)] + prefix = fp[:prefix_len] if use_prefix else "" + cand_groups = [] + if use_prefix: + for key, groups_for_key in prefix_map.items(): + if sdf_mod.prefix_distance(prefix, key) <= sdf_mod.PREFIX_THRESHOLD: + for g in groups_for_key: + cand_groups.append((g, key)) + else: + for g in prefix_map.get(prefix, []): + cand_groups.append((g, prefix)) + placed = False + for g, _key in cand_groups: + dist = sdf_mod.fingerprint_distance(fp, g["fp"]) + if dist <= threshold: + g["paths"].append(path) + placed = True + break + if not placed: + g = {"fp": fp, "paths": [path]} + prefix_map.setdefault(prefix, []).append(g) + groups.append(g) + pairs = set() + for g in groups: + paths = g["paths"] + if len(paths) <= 1: + continue + scored = sorted( + paths, key=lambda p: sdf_mod._keep_score(p, sdf_mod.EXT_PRIORITY), reverse=True + ) + keep = scored[0] + for dup in scored[1:]: + pairs.add(tuple(sorted((keep, dup)))) + return pairs + + def test_duplicate_detection(tmp_path, monkeypatch): sdf_mod = load_module(monkeypatch) (tmp_path / 'a.mp3').write_text('x') (tmp_path / 'b.flac').write_text('x') (tmp_path / 'c.mp3').write_text('x') db = tmp_path / 'fp.db' - dups, missing = sdf_mod.find_duplicates(str(tmp_path), db_path=str(db)) + _ensure_db(str(db)) + dups, missing = sdf_mod.find_duplicates(str(tmp_path), db_path=str(db), max_workers=1) pair = (str(tmp_path / 'b.flac'), str(tmp_path / 'a.mp3')) assert pair in dups or (pair[1], pair[0]) in dups assert missing == 0 @@ -41,9 +83,12 @@ def test_cross_prefix_detection(tmp_path, monkeypatch): (tmp_path / 'd.mp3').write_text('x') (tmp_path / 'e.mp3').write_text('x') db = tmp_path / 'fp.db' + _ensure_db(str(db)) # Fuzzy prefix grouping should still compare them pair = (str(tmp_path / 'd.mp3'), str(tmp_path / 'e.mp3')) - dups, _ = sdf_mod.find_duplicates(str(tmp_path), db_path=str(db), threshold=0.5) + dups, _ = sdf_mod.find_duplicates( + str(tmp_path), db_path=str(db), threshold=0.5, max_workers=1 + ) assert pair in dups or (pair[1], pair[0]) in dups flush_cache(str(db)) @@ -57,8 +102,34 @@ def test_fuzzy_prefix_grouping(tmp_path, monkeypatch): (tmp_path / 'f1.mp3').write_text('x') (tmp_path / 'f2.flac').write_text('x') db = tmp_path / 'fp.db' + _ensure_db(str(db)) pair = (str(tmp_path / 'f2.flac'), str(tmp_path / 'f1.mp3')) - dups, _ = sdf_mod.find_duplicates(str(tmp_path), db_path=str(db)) + dups, _ = sdf_mod.find_duplicates(str(tmp_path), db_path=str(db), max_workers=1) assert pair in dups or (pair[1], pair[0]) in dups flush_cache(str(db)) + +def test_coarse_gating_regression(tmp_path, monkeypatch): + fp_map = { + 'dup1.mp3': '1 2 3 4 5 6 7 8 9 10', + 'dup2.flac': '1 2 3 4 5 6 7 8 9 10', + 'near.mp3': '1 2 3 4 5 6 7 8 9 11', + 'far.mp3': '9 9 9 9 9 9 9 9 9 9', + } + sdf_mod = load_module(monkeypatch, fp_map=fp_map) + for name in fp_map: + (tmp_path / name).write_text('x') + db = tmp_path / 'fp.db' + _ensure_db(str(db)) + ordered_paths = [str(tmp_path / name) for name in sorted(fp_map)] + monkeypatch.setattr(sdf_mod, '_walk_audio_files', lambda *_args, **_kw: iter(ordered_paths)) + + expected = _naive_duplicate_pairs( + sdf_mod, ordered_paths, fp_map, prefix_len=sdf_mod.FP_PREFIX_LEN + ) + dups, _ = sdf_mod.find_duplicates( + str(tmp_path), db_path=str(db), max_workers=1, threshold=0.2 + ) + actual = {tuple(sorted(pair)) for pair in dups} + assert expected.issubset(actual) + flush_cache(str(db)) From 2f1be54d3b9fea0ede64130c9edde34e92732e0d Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 6 Jan 2026 22:53:21 -0500 Subject: [PATCH 310/606] Revert "Add coarse fingerprint gating to simple_duplicate_finder" --- simple_duplicate_finder.py | 37 +++---------- tests/test_simple_duplicate_finder.py | 79 ++------------------------- 2 files changed, 12 insertions(+), 104 deletions(-) diff --git a/simple_duplicate_finder.py b/simple_duplicate_finder.py index 0f5ef83..c146c30 100644 --- a/simple_duplicate_finder.py +++ b/simple_duplicate_finder.py @@ -5,7 +5,7 @@ from typing import Callable, List, Tuple, Dict, Optional, Iterator from fingerprint_cache import get_fingerprint -from near_duplicate_detector import fingerprint_distance, _coarse_fingerprint_keys, _coarse_gate +from near_duplicate_detector import fingerprint_distance import chromaprint_utils from config import load_config, FP_DURATION_MS, FP_OFFSET_MS, FP_SILENCE_MIN_LEN_MS, FP_SILENCE_THRESHOLD_DB @@ -148,39 +148,20 @@ def compute(path: str) -> Tuple[Optional[int], Optional[str]]: groups: List[Dict[str, object]] = [] prefix_map: Dict[str, List[Dict[str, object]]] = {} - coarse_index: Dict[str, List[Dict[str, object]]] = {} use_prefix = prefix_len is not None and prefix_len > 0 def handle_fingerprint(path: str, fp: str) -> None: prefix = fp[:prefix_len] if use_prefix else "" log_callback(f"[GROUP] path={path}, prefix={prefix}") - coarse_keys = _coarse_fingerprint_keys(fp) - candidate_groups: Dict[int, Dict[str, object]] = {} - if coarse_keys: - for key in coarse_keys: - for g in coarse_index.get(key, []): - candidate_groups[id(g)] = g - else: - for groups_for_key in prefix_map.values(): - for g in groups_for_key: - candidate_groups[id(g)] = g - - coarse_candidate_count = len(candidate_groups) cand_groups: List[Tuple[Dict[str, object], str]] = [] if use_prefix: - for g in candidate_groups.values(): - group_prefix = g.get("prefix", "") - if not coarse_keys or _coarse_gate(coarse_keys, g.get("coarse_keys", [])): - if prefix_distance(prefix, group_prefix) <= PREFIX_THRESHOLD: - cand_groups.append((g, group_prefix)) + for key, groups_for_key in prefix_map.items(): + if prefix_distance(prefix, key) <= PREFIX_THRESHOLD: + for g in groups_for_key: + cand_groups.append((g, key)) else: - for g in candidate_groups.values(): - if _coarse_gate(coarse_keys, g.get("coarse_keys", [])) or not coarse_keys: - cand_groups.append((g, g.get("prefix", ""))) - log_callback( - f"[COARSE] {path} coarse candidates {coarse_candidate_count} -> " - f"prefix candidates {len(cand_groups)}" - ) + for g in prefix_map.get(prefix, []): + cand_groups.append((g, prefix)) _dlog("GROUP", f"file {path} -> prefix {prefix}") _dlog("GROUP", f"{len(cand_groups)} groups for prefix") placed = False @@ -200,11 +181,9 @@ def handle_fingerprint(path: str, fp: str) -> None: placed = True break if not placed: - g = {"fp": fp, "paths": [path], "prefix": prefix, "coarse_keys": coarse_keys} + g = {"fp": fp, "paths": [path]} prefix_map.setdefault(prefix, []).append(g) groups.append(g) - for key in coarse_keys: - coarse_index.setdefault(key, []).append(g) _dlog("GROUP", f"new group for prefix {prefix}") with ThreadPoolExecutor(max_workers=max_workers) as executor: diff --git a/tests/test_simple_duplicate_finder.py b/tests/test_simple_duplicate_finder.py index d884461..64e64ef 100644 --- a/tests/test_simple_duplicate_finder.py +++ b/tests/test_simple_duplicate_finder.py @@ -4,7 +4,7 @@ import importlib import simple_duplicate_finder as sdf -from fingerprint_cache import flush_cache, _ensure_db +from fingerprint_cache import flush_cache def load_module(monkeypatch, fp_map=None): @@ -23,55 +23,13 @@ def load_module(monkeypatch, fp_map=None): return sdf -def _naive_duplicate_pairs(sdf_mod, ordered_paths, fp_map, prefix_len=None, threshold=0.03): - groups = [] - prefix_map = {} - use_prefix = prefix_len is not None and prefix_len > 0 - for path in ordered_paths: - fp = fp_map[os.path.basename(path)] - prefix = fp[:prefix_len] if use_prefix else "" - cand_groups = [] - if use_prefix: - for key, groups_for_key in prefix_map.items(): - if sdf_mod.prefix_distance(prefix, key) <= sdf_mod.PREFIX_THRESHOLD: - for g in groups_for_key: - cand_groups.append((g, key)) - else: - for g in prefix_map.get(prefix, []): - cand_groups.append((g, prefix)) - placed = False - for g, _key in cand_groups: - dist = sdf_mod.fingerprint_distance(fp, g["fp"]) - if dist <= threshold: - g["paths"].append(path) - placed = True - break - if not placed: - g = {"fp": fp, "paths": [path]} - prefix_map.setdefault(prefix, []).append(g) - groups.append(g) - pairs = set() - for g in groups: - paths = g["paths"] - if len(paths) <= 1: - continue - scored = sorted( - paths, key=lambda p: sdf_mod._keep_score(p, sdf_mod.EXT_PRIORITY), reverse=True - ) - keep = scored[0] - for dup in scored[1:]: - pairs.add(tuple(sorted((keep, dup)))) - return pairs - - def test_duplicate_detection(tmp_path, monkeypatch): sdf_mod = load_module(monkeypatch) (tmp_path / 'a.mp3').write_text('x') (tmp_path / 'b.flac').write_text('x') (tmp_path / 'c.mp3').write_text('x') db = tmp_path / 'fp.db' - _ensure_db(str(db)) - dups, missing = sdf_mod.find_duplicates(str(tmp_path), db_path=str(db), max_workers=1) + dups, missing = sdf_mod.find_duplicates(str(tmp_path), db_path=str(db)) pair = (str(tmp_path / 'b.flac'), str(tmp_path / 'a.mp3')) assert pair in dups or (pair[1], pair[0]) in dups assert missing == 0 @@ -83,12 +41,9 @@ def test_cross_prefix_detection(tmp_path, monkeypatch): (tmp_path / 'd.mp3').write_text('x') (tmp_path / 'e.mp3').write_text('x') db = tmp_path / 'fp.db' - _ensure_db(str(db)) # Fuzzy prefix grouping should still compare them pair = (str(tmp_path / 'd.mp3'), str(tmp_path / 'e.mp3')) - dups, _ = sdf_mod.find_duplicates( - str(tmp_path), db_path=str(db), threshold=0.5, max_workers=1 - ) + dups, _ = sdf_mod.find_duplicates(str(tmp_path), db_path=str(db), threshold=0.5) assert pair in dups or (pair[1], pair[0]) in dups flush_cache(str(db)) @@ -102,34 +57,8 @@ def test_fuzzy_prefix_grouping(tmp_path, monkeypatch): (tmp_path / 'f1.mp3').write_text('x') (tmp_path / 'f2.flac').write_text('x') db = tmp_path / 'fp.db' - _ensure_db(str(db)) pair = (str(tmp_path / 'f2.flac'), str(tmp_path / 'f1.mp3')) - dups, _ = sdf_mod.find_duplicates(str(tmp_path), db_path=str(db), max_workers=1) + dups, _ = sdf_mod.find_duplicates(str(tmp_path), db_path=str(db)) assert pair in dups or (pair[1], pair[0]) in dups flush_cache(str(db)) - -def test_coarse_gating_regression(tmp_path, monkeypatch): - fp_map = { - 'dup1.mp3': '1 2 3 4 5 6 7 8 9 10', - 'dup2.flac': '1 2 3 4 5 6 7 8 9 10', - 'near.mp3': '1 2 3 4 5 6 7 8 9 11', - 'far.mp3': '9 9 9 9 9 9 9 9 9 9', - } - sdf_mod = load_module(monkeypatch, fp_map=fp_map) - for name in fp_map: - (tmp_path / name).write_text('x') - db = tmp_path / 'fp.db' - _ensure_db(str(db)) - ordered_paths = [str(tmp_path / name) for name in sorted(fp_map)] - monkeypatch.setattr(sdf_mod, '_walk_audio_files', lambda *_args, **_kw: iter(ordered_paths)) - - expected = _naive_duplicate_pairs( - sdf_mod, ordered_paths, fp_map, prefix_len=sdf_mod.FP_PREFIX_LEN - ) - dups, _ = sdf_mod.find_duplicates( - str(tmp_path), db_path=str(db), max_workers=1, threshold=0.2 - ) - actual = {tuple(sorted(pair)) for pair in dups} - assert expected.issubset(actual) - flush_cache(str(db)) From b8007cdfb35a8958d7dc8e93408ae2040881f239 Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Tue, 6 Jan 2026 22:58:20 -0500 Subject: [PATCH 311/606] Fix coarse gating candidate selection --- simple_duplicate_finder.py | 37 ++++++++++--- tests/test_simple_duplicate_finder.py | 79 +++++++++++++++++++++++++-- 2 files changed, 104 insertions(+), 12 deletions(-) diff --git a/simple_duplicate_finder.py b/simple_duplicate_finder.py index c146c30..e277a43 100644 --- a/simple_duplicate_finder.py +++ b/simple_duplicate_finder.py @@ -5,7 +5,7 @@ from typing import Callable, List, Tuple, Dict, Optional, Iterator from fingerprint_cache import get_fingerprint -from near_duplicate_detector import fingerprint_distance +from near_duplicate_detector import fingerprint_distance, _coarse_fingerprint_keys import chromaprint_utils from config import load_config, FP_DURATION_MS, FP_OFFSET_MS, FP_SILENCE_MIN_LEN_MS, FP_SILENCE_THRESHOLD_DB @@ -148,20 +148,36 @@ def compute(path: str) -> Tuple[Optional[int], Optional[str]]: groups: List[Dict[str, object]] = [] prefix_map: Dict[str, List[Dict[str, object]]] = {} + coarse_index: Dict[str, List[Dict[str, object]]] = {} + ungated_groups: List[Dict[str, object]] = [] use_prefix = prefix_len is not None and prefix_len > 0 def handle_fingerprint(path: str, fp: str) -> None: prefix = fp[:prefix_len] if use_prefix else "" log_callback(f"[GROUP] path={path}, prefix={prefix}") + coarse_keys = _coarse_fingerprint_keys(fp) + candidate_groups: Dict[int, Dict[str, object]] = {} + if coarse_keys: + for key in coarse_keys: + for g in coarse_index.get(key, []): + candidate_groups[id(g)] = g + for g in ungated_groups: + candidate_groups[id(g)] = g + + coarse_candidate_count = len(candidate_groups) cand_groups: List[Tuple[Dict[str, object], str]] = [] if use_prefix: - for key, groups_for_key in prefix_map.items(): - if prefix_distance(prefix, key) <= PREFIX_THRESHOLD: - for g in groups_for_key: - cand_groups.append((g, key)) + for g in candidate_groups.values(): + group_prefix = g.get("prefix", "") + if prefix_distance(prefix, group_prefix) <= PREFIX_THRESHOLD: + cand_groups.append((g, group_prefix)) else: - for g in prefix_map.get(prefix, []): - cand_groups.append((g, prefix)) + for g in candidate_groups.values(): + cand_groups.append((g, g.get("prefix", ""))) + log_callback( + f"[COARSE] {path} coarse candidates {coarse_candidate_count} -> " + f"prefix candidates {len(cand_groups)}" + ) _dlog("GROUP", f"file {path} -> prefix {prefix}") _dlog("GROUP", f"{len(cand_groups)} groups for prefix") placed = False @@ -181,9 +197,14 @@ def handle_fingerprint(path: str, fp: str) -> None: placed = True break if not placed: - g = {"fp": fp, "paths": [path]} + g = {"fp": fp, "paths": [path], "prefix": prefix} prefix_map.setdefault(prefix, []).append(g) groups.append(g) + if coarse_keys: + for key in coarse_keys: + coarse_index.setdefault(key, []).append(g) + else: + ungated_groups.append(g) _dlog("GROUP", f"new group for prefix {prefix}") with ThreadPoolExecutor(max_workers=max_workers) as executor: diff --git a/tests/test_simple_duplicate_finder.py b/tests/test_simple_duplicate_finder.py index 64e64ef..d884461 100644 --- a/tests/test_simple_duplicate_finder.py +++ b/tests/test_simple_duplicate_finder.py @@ -4,7 +4,7 @@ import importlib import simple_duplicate_finder as sdf -from fingerprint_cache import flush_cache +from fingerprint_cache import flush_cache, _ensure_db def load_module(monkeypatch, fp_map=None): @@ -23,13 +23,55 @@ def load_module(monkeypatch, fp_map=None): return sdf +def _naive_duplicate_pairs(sdf_mod, ordered_paths, fp_map, prefix_len=None, threshold=0.03): + groups = [] + prefix_map = {} + use_prefix = prefix_len is not None and prefix_len > 0 + for path in ordered_paths: + fp = fp_map[os.path.basename(path)] + prefix = fp[:prefix_len] if use_prefix else "" + cand_groups = [] + if use_prefix: + for key, groups_for_key in prefix_map.items(): + if sdf_mod.prefix_distance(prefix, key) <= sdf_mod.PREFIX_THRESHOLD: + for g in groups_for_key: + cand_groups.append((g, key)) + else: + for g in prefix_map.get(prefix, []): + cand_groups.append((g, prefix)) + placed = False + for g, _key in cand_groups: + dist = sdf_mod.fingerprint_distance(fp, g["fp"]) + if dist <= threshold: + g["paths"].append(path) + placed = True + break + if not placed: + g = {"fp": fp, "paths": [path]} + prefix_map.setdefault(prefix, []).append(g) + groups.append(g) + pairs = set() + for g in groups: + paths = g["paths"] + if len(paths) <= 1: + continue + scored = sorted( + paths, key=lambda p: sdf_mod._keep_score(p, sdf_mod.EXT_PRIORITY), reverse=True + ) + keep = scored[0] + for dup in scored[1:]: + pairs.add(tuple(sorted((keep, dup)))) + return pairs + + def test_duplicate_detection(tmp_path, monkeypatch): sdf_mod = load_module(monkeypatch) (tmp_path / 'a.mp3').write_text('x') (tmp_path / 'b.flac').write_text('x') (tmp_path / 'c.mp3').write_text('x') db = tmp_path / 'fp.db' - dups, missing = sdf_mod.find_duplicates(str(tmp_path), db_path=str(db)) + _ensure_db(str(db)) + dups, missing = sdf_mod.find_duplicates(str(tmp_path), db_path=str(db), max_workers=1) pair = (str(tmp_path / 'b.flac'), str(tmp_path / 'a.mp3')) assert pair in dups or (pair[1], pair[0]) in dups assert missing == 0 @@ -41,9 +83,12 @@ def test_cross_prefix_detection(tmp_path, monkeypatch): (tmp_path / 'd.mp3').write_text('x') (tmp_path / 'e.mp3').write_text('x') db = tmp_path / 'fp.db' + _ensure_db(str(db)) # Fuzzy prefix grouping should still compare them pair = (str(tmp_path / 'd.mp3'), str(tmp_path / 'e.mp3')) - dups, _ = sdf_mod.find_duplicates(str(tmp_path), db_path=str(db), threshold=0.5) + dups, _ = sdf_mod.find_duplicates( + str(tmp_path), db_path=str(db), threshold=0.5, max_workers=1 + ) assert pair in dups or (pair[1], pair[0]) in dups flush_cache(str(db)) @@ -57,8 +102,34 @@ def test_fuzzy_prefix_grouping(tmp_path, monkeypatch): (tmp_path / 'f1.mp3').write_text('x') (tmp_path / 'f2.flac').write_text('x') db = tmp_path / 'fp.db' + _ensure_db(str(db)) pair = (str(tmp_path / 'f2.flac'), str(tmp_path / 'f1.mp3')) - dups, _ = sdf_mod.find_duplicates(str(tmp_path), db_path=str(db)) + dups, _ = sdf_mod.find_duplicates(str(tmp_path), db_path=str(db), max_workers=1) assert pair in dups or (pair[1], pair[0]) in dups flush_cache(str(db)) + +def test_coarse_gating_regression(tmp_path, monkeypatch): + fp_map = { + 'dup1.mp3': '1 2 3 4 5 6 7 8 9 10', + 'dup2.flac': '1 2 3 4 5 6 7 8 9 10', + 'near.mp3': '1 2 3 4 5 6 7 8 9 11', + 'far.mp3': '9 9 9 9 9 9 9 9 9 9', + } + sdf_mod = load_module(monkeypatch, fp_map=fp_map) + for name in fp_map: + (tmp_path / name).write_text('x') + db = tmp_path / 'fp.db' + _ensure_db(str(db)) + ordered_paths = [str(tmp_path / name) for name in sorted(fp_map)] + monkeypatch.setattr(sdf_mod, '_walk_audio_files', lambda *_args, **_kw: iter(ordered_paths)) + + expected = _naive_duplicate_pairs( + sdf_mod, ordered_paths, fp_map, prefix_len=sdf_mod.FP_PREFIX_LEN + ) + dups, _ = sdf_mod.find_duplicates( + str(tmp_path), db_path=str(db), max_workers=1, threshold=0.2 + ) + actual = {tuple(sorted(pair)) for pair in dups} + assert expected.issubset(actual) + flush_cache(str(db)) From 05953fd28f3d44e570ba54c6ac510561b4d6df2a Mon Sep 17 00:00:00 2001 From: Greensand321 Date: Wed, 7 Jan 2026 19:14:04 -0500 Subject: [PATCH 312/606] Update indexer documentation --- README.md | 17 +++++++++--- docs/project_documentation.html | 49 ++++++++++++++++----------------- 2 files changed, 37 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index b060b53..264a22f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # SoundVault Music Indexer -SoundVault organizes large music libraries. It deduplicates tracks, fixes tags via AcoustID, normalizes genres, and generates playlists while keeping your folder structure intact. +SoundVault organizes large music libraries. It deduplicates tracks, fixes tags via AcoustID, normalizes genres, and generates playlists with preview-first workflows before moving or renaming files. ## Prerequisites @@ -56,7 +56,7 @@ python main_gui.py ``` 1. **Open** your library folder -2. Use the **Indexer** tab to dedupe, detect near duplicates, and move files +2. Use the **Indexer** tab to preview and organize files; dry runs open the HTML preview, and full runs apply the move plan. 3. **Fix Tags** via the AcoustID menu (now supports multiple metadata services) 4. **Generate Playlists** from your folder structure 5. **Clustered Playlists** (interactive K-Means/HDBSCAN) via the Tools ▸ Clustered Playlists menu @@ -64,7 +64,7 @@ python main_gui.py 7. **Auto‑DJ** mode builds seamless playlists starting from any song 8. Use the **Library Sync** tab to compare an incoming folder against your existing library, then build/preview/execute an optional copy/move plan. 9. Launch the **Duplicate Finder** tab to open the updated shell for spotting duplicates. -10. **Cross-Album Scan** optionally finds duplicates appearing on multiple albums (Indexer tab → Phase 3). +10. **Cross-Album Scan** is a reserved toggle in the Indexer UI; duplicate detection lives in the **Duplicate Finder** tab. 11. Use the **Theme** dropdown and **Help** tab for assistance. 12. Use **Tools → Similarity Inspector** to compare two files and see fingerprint distance details. @@ -78,6 +78,15 @@ Cluster generation writes progress into `_log.txt` inside your library s The indexer automatically prefixes file paths with `\\?\` on Windows, allowing it to work with directories deeper than the classic 260-character limit. +### Indexer outputs and behavior + +- **Preview output:** Every run writes `Docs/MusicIndex.html` under the selected library root; dry runs open the preview automatically. +- **Decision log:** A detailed `Docs/indexer_log.txt` captures routing decisions and metadata fallbacks. +- **Missing metadata:** Tracks missing core tags land in `Manual Review/` so you can fix them before re-running. +- **Excluded folders:** `Not Sorted/` and `Playlists/` are skipped during scans, letting you stash exceptions safely. +- **Leftover files:** Non-audio leftovers are moved into `Docs/` (`.txt`, `.html`, `.db`) or `Trash/` for everything else. +- **Playlists:** Full runs generate playlists in `Playlists/` by default; uncheck **Create Playlists** to skip. + ## Threading Long running actions such as indexing, tag fixing and library sync operations are executed in daemon threads. GUI updates from these background tasks are scheduled using Tkinter's `after` method so message boxes and progress indicators always run on the main thread. @@ -175,7 +184,7 @@ The codebase is organized into a handful of key modules: ``` main_gui.py - Tkinter entry point for the desktop app -music_indexer_api.py - Core scanning, dedupe and relocation logic +music_indexer_api.py - Core scanning and relocation logic (dedupe handled elsewhere) playlist_generator.py - `.m3u` playlist creation helpers clustered_playlists.py - Feature extraction and clustering algorithms cluster_graph_panel.py - Interactive scatter plot for clustered playlists diff --git a/docs/project_documentation.html b/docs/project_documentation.html index 7cb2446..a286a8d 100644 --- a/docs/project_documentation.html +++ b/docs/project_documentation.html @@ -34,10 +34,11 @@

      2.1 Music Indexer (music_indexer_api.py)

      • Scans folders recursively for audio files.
      • Reads metadata: artist, title, album, year, track number, embedded cover art.
      • -
      • Deduplicates tracks by fingerprint, preferring FLAC > M4A > MP3.
      • -
      • Separates workflow into deterministic phases: vault pre-scan (artist counts), fingerprint-based dedupe, metadata aggregation, destination routing, and preview generation.
      • +
      • Separates workflow into deterministic phases: vault pre-scan (artist counts), metadata aggregation, destination routing, and preview generation.
      • Surfaces every routing decision in decision_log and a tag index to drive consistent HTML previews and downstream plan rendering.
      • Phase 0: Pre-scan the vault under MUSIC_ROOT to build a global artist-count map via build_primary_counts().
      • +
      • Skips Not Sorted/ and Playlists/ directories during scans so you can exclude files from indexing.
      • +
      • Routes missing-metadata tracks into Manual Review/ for later cleanup.
      • Implements grouping rules:
        • Common artist threshold (≥ 10 tracks) → By Artist/<Artist>.
        • @@ -46,9 +47,10 @@

          2.1 Music Indexer (music_indexer_api.py)

        • “(Remixes)” folder if an album has ≥ 3 remixes.
      • -
      • Generates a “dry-run” HTML preview of the proposed reorganization (color-coded).
      • -
      • Applies the final moves: creates new folders, renames/moves audio files, archives docs into Docs/ and other leftovers into Trash/, then deletes empty folders (two passes to also remove now-empty parents).
      • -
      • Writes Docs/indexer_log.txt detailing exactly how each file was classified and moved.
      • +
      • Generates a “dry-run” HTML preview of the proposed reorganization (color-coded) at Docs/MusicIndex.html.
      • +
      • Applies the final moves: creates new folders, renames/moves audio files, updates canonical primary-artist metadata, and archives leftovers into Docs/ or Trash/, then deletes empty folders (two passes to also remove now-empty parents).
      • +
      • Writes Docs/indexer_log.txt detailing exactly how each file was classified and routed.
      • +
      • Note: the indexer does not deduplicate; duplicate review lives in the Duplicate Finder workflow.

      Skeleton of music_indexer_api.py (comments only):

      @@ -79,21 +81,17 @@

      2.1 Music Indexer (music_indexer_api.py)

      def compute_moves_and_tag_index(root_path, log_callback=None): # 0. Pre-scan entire vault under MUSIC_ROOT -> build_primary_counts() # 1. Decide music_root (either root_path or root_path/Music). - # 2. Walk recursively, collect supported audio files. - # 3. Deduplicate by (artist, title, album), choose highest-quality format. - # 4. Read metadata for each kept track; build counts: - # • primary_counts: number of tracks per artist + # 2. Walk recursively, collect supported audio files (skip Not Sorted/Playlists). + # 3. Read metadata for each track; build counts: # • album_counts: number of tracks per (artist, album) - # • remix_counts: number of remixed tracks per (artist, album) - # • cover_counts: number of identical embedded-cover hashes - # 5. For each file, decide destination: - # • If "(Remixes)" and remix_count ≥ REMIX_FOLDER_THRESHOLD → create " (Remixes)" folder - # • Else if remix in title but remix_count < threshold → treat as single - # • Else if album_count ≥ 5 → create album folder - # • Else if primary_count ≥ COMMON_ARTIST_THRESHOLD → create "/Album" or " – Singles" - # • Else → / ... - # 6. Build a moves dict mapping old_path → new_path, plus a tag_index for HTML preview. - # 7. Return (moves, tag_index, decision_log). + # • remix_counts: number of remix-tagged tracks per (artist, album) + # • album_disc_values: disc values per (artist, album) + # 4. For each file, decide destination: + # • If missing core tags → Manual Review/ + # • If rare artist → By Year/ + # • Else apply album/singles/remix grouping rules + # 5. Build a moves dict mapping old_path → new_path, plus a tag_index for HTML preview. + # 6. Return (moves, tag_index, decision_log). pass def build_dry_run_html(root_path, output_html_path, log_callback=None): @@ -105,10 +103,10 @@

      2.1 Music Indexer (music_indexer_api.py)

      # Actually performs the file moves: # 1. Creates new directories as needed. # 2. Moves and renames audio files. - # 3. Moves docs (*.txt, *.html, *.db) into "Docs/" and other leftovers into "Trash/". - # 4. Deletes empty source folders in two passes so parent directories left + # 3. Updates canonical primary-artist metadata where possible. + # 4. Moves docs (*.txt, *.html, *.db) into "Docs/" and other leftovers into "Trash/". + # 5. Deletes empty source folders in two passes so parent directories left # empty after their children are removed are also deleted. - # 5. Writes Docs/indexer_log.txt summarizing each file’s action. pass def run_full_indexer(root_path, output_html_path, dry_run_only=False, log_callback=None): @@ -117,6 +115,7 @@

      2.1 Music Indexer (music_indexer_api.py)

      # 2. Write Docs/indexer_log.txt # 3. build_dry_run_html(...) # 4. If not dry_run_only, call apply_indexer_moves(...) + # 5. Optionally generate playlists under Playlists/ pass
    @@ -573,13 +572,13 @@

    2.y GUI Front-end (main_gui.py)

    3. Current Progress