diff --git a/pkg/api/options.go b/pkg/api/options.go index d6b6b9c6a..f331da90b 100644 --- a/pkg/api/options.go +++ b/pkg/api/options.go @@ -186,6 +186,14 @@ type RequestSaveOptionsTaskSchedule struct { LinkScenesHourStart int `json:"linkScenesHourStart"` LinkScenesHourEnd int `json:"linkScenesHourEnd"` LinkScenesStartDelay int `json:"linkScenesStartDelay"` + + OrganizeEnabled bool `json:"organizeEnabled"` + OrganizeHourInterval int `json:"organizeHourInterval"` + OrganizeUseRange bool `json:"organizeUseRange"` + OrganizeMinuteStart int `json:"organizeMinuteStart"` + OrganizeHourStart int `json:"organizeHourStart"` + OrganizeHourEnd int `json:"organizeHourEnd"` + OrganizeStartDelay int `json:"organizeStartDelay"` } type RequestSaveSiteMatchParams struct { SiteId string `json:"site"` @@ -1038,6 +1046,14 @@ func (i ConfigResource) saveOptionsTaskSchedule(req *restful.Request, resp *rest config.Config.Cron.LinkScenesSchedule.HourEnd = r.LinkScenesHourEnd config.Config.Cron.LinkScenesSchedule.RunAtStartDelay = r.LinkScenesStartDelay + config.Config.Cron.OrganizeSchedule.Enabled = r.OrganizeEnabled + config.Config.Cron.OrganizeSchedule.HourInterval = r.OrganizeHourInterval + config.Config.Cron.OrganizeSchedule.UseRange = r.OrganizeUseRange + config.Config.Cron.OrganizeSchedule.MinuteStart = r.OrganizeMinuteStart + config.Config.Cron.OrganizeSchedule.HourStart = r.OrganizeHourStart + config.Config.Cron.OrganizeSchedule.HourEnd = r.OrganizeHourEnd + config.Config.Cron.OrganizeSchedule.RunAtStartDelay = r.OrganizeStartDelay + config.SaveConfig() resp.WriteHeaderAndEntity(http.StatusOK, r) diff --git a/pkg/api/organize.go b/pkg/api/organize.go new file mode 100644 index 000000000..2b58e5dfa --- /dev/null +++ b/pkg/api/organize.go @@ -0,0 +1,180 @@ +package api + +import ( + "net/http" + + restfulspec "github.com/emicklei/go-restful-openapi/v2" + "github.com/emicklei/go-restful/v3" + + "github.com/xbapps/xbvr/pkg/config" + "github.com/xbapps/xbvr/pkg/models" + "github.com/xbapps/xbvr/pkg/organize" +) + +type OrganizeResource struct{} + +type RequestOrganizeRun struct { + DryRun bool `json:"dryRun"` + Limit int `json:"limit"` +} + +type RequestOrganizeConfig struct { + Dedup bool `json:"dedup"` + DeferDups bool `json:"deferDups"` + IncomingDir string `json:"incomingDir"` + IncomingMinAge int `json:"incomingMinAge"` + TopFolder string `json:"topFolder"` + CastGender string `json:"castGender"` + SymlinkByActor bool `json:"symlinkByActor"` + ActorFolder string `json:"actorFolder"` +} + +func (i OrganizeResource) WebService() *restful.WebService { + tags := []string{"Organize"} + ws := new(restful.WebService) + ws.Path("/api/organize").Consumes(restful.MIME_JSON).Produces(restful.MIME_JSON) + + ws.Route(ws.POST("/run").To(i.run).Metadata(restfulspec.KeyOpenAPITags, tags)) + ws.Route(ws.GET("/status").To(i.status).Metadata(restfulspec.KeyOpenAPITags, tags)) + ws.Route(ws.GET("/config").To(i.getConfig).Metadata(restfulspec.KeyOpenAPITags, tags)) + ws.Route(ws.POST("/config").To(i.saveConfig).Metadata(restfulspec.KeyOpenAPITags, tags)) + + ws.Route(ws.POST("/duplicates/analyze").Consumes("*/*").To(i.dupAnalyze).Metadata(restfulspec.KeyOpenAPITags, tags)) + ws.Route(ws.GET("/duplicates").To(i.dupList).Metadata(restfulspec.KeyOpenAPITags, tags)) + ws.Route(ws.POST("/duplicates/ignore").To(i.dupIgnore).Metadata(restfulspec.KeyOpenAPITags, tags)) + ws.Route(ws.POST("/duplicates/unignore").To(i.dupUnignore).Metadata(restfulspec.KeyOpenAPITags, tags)) + ws.Route(ws.POST("/duplicates/delete").To(i.dupDelete).Metadata(restfulspec.KeyOpenAPITags, tags)) + ws.Route(ws.POST("/duplicates/disassociate").To(i.dupDisassociate).Metadata(restfulspec.KeyOpenAPITags, tags)) + return ws +} + +func (i OrganizeResource) dupAnalyze(req *restful.Request, resp *restful.Response) { + force := req.QueryParameter("force") == "true" + if organize.StartDupAnalysis(force) { + resp.WriteHeaderAndEntity(http.StatusOK, map[string]string{"status": "started"}) + } else { + resp.WriteHeaderAndEntity(http.StatusOK, map[string]string{"status": "already-running"}) + } +} + +func (i OrganizeResource) dupList(req *restful.Request, resp *restful.Response) { + db, _ := models.GetDB() + defer db.Close() + running, done, total := organize.DupStatus() + showIgnored := req.QueryParameter("showIgnored") == "true" + resp.WriteHeaderAndEntity(http.StatusOK, map[string]interface{}{ + "running": running, "done": done, "total": total, + "groups": organize.ListDupGroups(db, showIgnored), + }) +} + +func (i OrganizeResource) dupIgnore(req *restful.Request, resp *restful.Response) { + i.dupSetIgnore(req, resp, true) +} +func (i OrganizeResource) dupUnignore(req *restful.Request, resp *restful.Response) { + i.dupSetIgnore(req, resp, false) +} +func (i OrganizeResource) dupSetIgnore(req *restful.Request, resp *restful.Response, ignore bool) { + var r struct { + FileID uint `json:"fileId"` + } + if err := req.ReadEntity(&r); err != nil { + resp.WriteHeaderAndEntity(http.StatusBadRequest, nil) + return + } + db, _ := models.GetDB() + defer db.Close() + if ignore { + organize.IgnoreFile(db, r.FileID) + } else { + organize.UnignoreFile(db, r.FileID) + } + resp.WriteHeaderAndEntity(http.StatusOK, map[string]bool{"ignored": ignore}) +} + +func (i OrganizeResource) dupDelete(req *restful.Request, resp *restful.Response) { + var r struct { + FileIDs []uint `json:"fileIds"` + } + if err := req.ReadEntity(&r); err != nil { + resp.WriteHeaderAndEntity(http.StatusBadRequest, nil) + return + } + db, _ := models.GetDB() + defer db.Close() + n := organize.DeleteFiles(db, r.FileIDs) + resp.WriteHeaderAndEntity(http.StatusOK, map[string]int{"deleted": n}) +} + +func (i OrganizeResource) dupDisassociate(req *restful.Request, resp *restful.Response) { + var r struct { + FileIDs []uint `json:"fileIds"` + } + if err := req.ReadEntity(&r); err != nil { + resp.WriteHeaderAndEntity(http.StatusBadRequest, nil) + return + } + db, _ := models.GetDB() + defer db.Close() + n := organize.DisassociateFiles(db, r.FileIDs) + resp.WriteHeaderAndEntity(http.StatusOK, map[string]int{"disassociated": n}) +} + +// optionsFromConfig builds run options from the persisted organize config. +func optionsFromConfig(dryRun bool, limit int) organize.Options { + c := config.Config.Organize + return organize.Options{ + DryRun: dryRun, + Limit: limit, + Dedup: c.Dedup, + DeferDups: c.DeferDups, + IncomingDir: c.IncomingDir, + IncomingMinAge: c.IncomingMinAge, + TopFolder: c.TopFolder, + CastGender: c.CastGender, + SymlinkByActor: c.SymlinkByActor, + ActorFolder: c.ActorFolder, + } +} + +func (i OrganizeResource) run(req *restful.Request, resp *restful.Response) { + var r RequestOrganizeRun + if err := req.ReadEntity(&r); err != nil { + log.Error(err) + resp.WriteHeaderAndEntity(http.StatusBadRequest, nil) + return + } + if organize.Start(optionsFromConfig(r.DryRun, r.Limit)) { + resp.WriteHeaderAndEntity(http.StatusOK, map[string]string{"status": "started"}) + } else { + resp.WriteHeaderAndEntity(http.StatusOK, map[string]string{"status": "already-running"}) + } +} + +func (i OrganizeResource) status(req *restful.Request, resp *restful.Response) { + running, result := organize.Status() + resp.WriteHeaderAndEntity(http.StatusOK, map[string]interface{}{"running": running, "result": result}) +} + +func (i OrganizeResource) getConfig(req *restful.Request, resp *restful.Response) { + resp.WriteHeaderAndEntity(http.StatusOK, config.Config.Organize) +} + +func (i OrganizeResource) saveConfig(req *restful.Request, resp *restful.Response) { + var r RequestOrganizeConfig + if err := req.ReadEntity(&r); err != nil { + log.Error(err) + resp.WriteHeaderAndEntity(http.StatusBadRequest, nil) + return + } + config.Config.Organize.Dedup = r.Dedup + config.Config.Organize.DeferDups = r.DeferDups + config.Config.Organize.IncomingDir = r.IncomingDir + config.Config.Organize.IncomingMinAge = r.IncomingMinAge + config.Config.Organize.TopFolder = r.TopFolder + config.Config.Organize.CastGender = r.CastGender + config.Config.Organize.SymlinkByActor = r.SymlinkByActor + config.Config.Organize.ActorFolder = r.ActorFolder + config.SaveConfig() + resp.WriteHeaderAndEntity(http.StatusOK, config.Config.Organize) +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 0a2a4a8d8..9202330e3 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -174,7 +174,26 @@ type ObjectConfig struct { HourEnd int `default:"23" json:"hourEnd"` RunAtStartDelay int `default:"0" json:"runAtStartDelay"` } `json:"linkScenesSchedule"` + OrganizeSchedule struct { + Enabled bool `default:"false" json:"enabled"` + HourInterval int `default:"24" json:"hourInterval"` + UseRange bool `default:"false" json:"useRange"` + MinuteStart int `default:"0" json:"minuteStart"` + HourStart int `default:"0" json:"hourStart"` + HourEnd int `default:"23" json:"hourEnd"` + RunAtStartDelay int `default:"0" json:"runAtStartDelay"` + } `json:"organizeSchedule"` } `json:"cron"` + Organize struct { + Dedup bool `default:"true" json:"dedup"` + DeferDups bool `default:"false" json:"deferDups"` + IncomingDir string `default:"Incoming" json:"incomingDir"` + IncomingMinAge int `default:"30" json:"incomingMinAge"` + TopFolder string `default:"" json:"topFolder"` + CastGender string `default:"female" json:"castGender"` + SymlinkByActor bool `default:"false" json:"symlinkByActor"` + ActorFolder string `default:"ByActor" json:"actorFolder"` + } `json:"organize"` Storage struct { MatchOhash bool `default:"false" json:"match_ohash"` VideoExt []string `json:"video_ext"` diff --git a/pkg/migrations/migrations.go b/pkg/migrations/migrations.go index 9d2a818b2..68f2ff692 100644 --- a/pkg/migrations/migrations.go +++ b/pkg/migrations/migrations.go @@ -25,6 +25,7 @@ import ( "github.com/xbapps/xbvr/pkg/common" "github.com/xbapps/xbvr/pkg/config" "github.com/xbapps/xbvr/pkg/models" + "github.com/xbapps/xbvr/pkg/organize" "github.com/xbapps/xbvr/pkg/scrape" "github.com/xbapps/xbvr/pkg/tasks" ) @@ -2387,6 +2388,18 @@ func Migrate(migrateTo string) { return tx.Table("scenes").AddIndex("idx_scenes_scraper_id", "scraper_id").Error }, }, + { + ID: "0088-duplicate-analysis", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(organize.DuplicateDismissal{}, organize.DuplicateReport{}).Error + }, + }, + { + ID: "0089-duplicate-file-ignore", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(organize.DuplicateFileIgnore{}).Error + }, + }, } // Wrap migrations to automatically track progress diff --git a/pkg/organize/apply.go b/pkg/organize/apply.go new file mode 100644 index 000000000..78815b0ca --- /dev/null +++ b/pkg/organize/apply.go @@ -0,0 +1,188 @@ +package organize + +import ( + "os" + "path/filepath" + "sort" + "strings" + + "github.com/jinzhu/gorm" + + "github.com/xbapps/xbvr/pkg/common" + "github.com/xbapps/xbvr/pkg/models" +) + +func applyPlan(db *gorm.DB, p *scenePlan, opts Options, res *Result, removedPaths, dryRemovedDirs map[string]bool) { + if !opts.DryRun { + os.MkdirAll(p.targetDir, 0o755) + } + + deletedRows := false + + // Identical-content duplicates first (both files still at source; md5 confirms). + for _, d := range p.contentDup { + rel, _ := filepath.Rel(p.targetDir, d.identicalTo) + if d.maybe { + // Dry-run candidate: md5 not verified, so don't count it as a real delete. + res.Actions = append(res.Actions, Action{SceneID: p.sceneID, Kind: "dedup-maybe", From: d.src, Note: "possible == " + rel + " (md5 unconfirmed)"}) + continue + } + if !opts.DryRun { + m1, ok1 := fileMD5(d.src) + m2, ok2 := fileMD5(d.identicalTo) + if !ok1 || !ok2 || m1 != m2 { + continue + } + if err := os.Remove(d.src); err != nil { + continue + } + db.Delete(&models.File{}, d.file.ID) + deletedRows = true + } + res.Actions = append(res.Actions, Action{SceneID: p.sceneID, Kind: "dedup-delete", From: d.src, Note: "== " + rel}) + res.Dedups++ + res.BytesReclaimed += d.size + } + + // Moves / renames of kept files. + for _, m := range p.moves { + if m.isMove { + kind := "move" + if filepath.Dir(m.src) == p.targetDir { + kind = "rename" + } + if !opts.DryRun { + if err := moveFile(m.src, m.dst); err != nil { + common.Log.Warnf("organize: move %s failed: %v", m.src, err) + continue + } + } + res.Actions = append(res.Actions, Action{SceneID: p.sceneID, Kind: kind, From: m.src, To: m.dst}) + if kind == "rename" { + res.FilesRenamed++ + } else { + res.FilesMoved++ + } + } + if !opts.DryRun && (m.file.Path != p.targetDir || m.file.Filename != m.name) { + db.Model(&models.File{}).Where("id = ?", m.file.ID). + Updates(map[string]interface{}{"path": p.targetDir, "filename": m.name}) + } + } + + // Hard-link duplicates: unlink, drop the stale DB row. + for _, f := range p.hlUnlink { + src := f.GetPath() + if !opts.DryRun { + os.Remove(src) + db.Delete(&models.File{}, f.ID) + deletedRows = true + } + res.Actions = append(res.Actions, Action{SceneID: p.sceneID, Kind: "hardlink-unlink", From: src}) + res.Hardlinks++ + } + + // Sidecars. + for _, s := range p.sidecars { + if !opts.DryRun { + if err := moveFile(s[0], s[1]); err != nil { + continue + } + db.Model(&models.File{}).Where("path = ? AND filename = ?", filepath.Dir(s[0]), filepath.Base(s[0])). + Updates(map[string]interface{}{"path": p.targetDir, "filename": filepath.Base(s[1])}) + } + res.Actions = append(res.Actions, Action{SceneID: p.sceneID, Kind: "sidecar", From: s[0], To: s[1]}) + res.Sidecars++ + } + + // Per-actor symlinks into CamelCase folders. + for _, s := range p.symlinks { + link, target := s[0], s[1] + if !opts.DryRun { + if err := os.MkdirAll(filepath.Dir(link), 0o755); err != nil { + common.Log.Warnf("organize: mkdir %s failed: %v", filepath.Dir(link), err) + continue + } + if err := os.Symlink(target, link); err != nil { + common.Log.Warnf("organize: symlink %s failed: %v", link, err) + continue + } + } + res.Actions = append(res.Actions, Action{SceneID: p.sceneID, Kind: "symlink", From: link, To: target}) + res.Symlinks++ + } + + // Record which source files left, for empty-dir detection. + for _, m := range p.moves { + if m.isMove { + removedPaths[m.src] = true + } + } + for _, f := range p.hlUnlink { + removedPaths[f.GetPath()] = true + } + for _, d := range p.contentDup { + if d.maybe { + continue // candidate only; not actually removed + } + removedPaths[d.src] = true + } + for _, s := range p.sidecars { + removedPaths[s[0]] = true + } + + // Remove emptied source directories, deepest first. + dirs := make([]string, 0, len(p.rmdirs)) + for d := range p.rmdirs { + dirs = append(dirs, d) + } + sort.Slice(dirs, func(i, j int) bool { return len(dirs[i]) > len(dirs[j]) }) + for _, d := range dirs { + if d == p.targetDir { + continue + } + removeEmptyDirs(d, p.targetDir, p.volumeRoot, opts, res, removedPaths, dryRemovedDirs) + } + + // Deleting a scene's own files can change its availability; refresh it. + if deletedRows { + var s models.Scene + if db.First(&s, p.sceneID).Error == nil { + s.UpdateStatus() + } + } +} + +func removeEmptyDirs(start, target, prefix string, opts Options, res *Result, removedPaths, dryRemovedDirs map[string]bool) { + if prefix == "" { + return + } + d := start + // Stop at the storage-folder root, and never walk into a directory that contains the + // target (the target keeps it non-empty once files land there). + for d != "" && d != prefix && strings.HasPrefix(d, prefix+"/") && !isUnder(target, d) { + entries, err := os.ReadDir(d) + if err != nil { + return + } + remaining := 0 + for _, e := range entries { + p := filepath.Join(d, e.Name()) + if opts.DryRun && (removedPaths[p] || dryRemovedDirs[p]) { + continue + } + remaining++ + } + if remaining > 0 { + return + } + if opts.DryRun { + dryRemovedDirs[d] = true + } else if err := os.Remove(d); err != nil { + return + } + res.Actions = append(res.Actions, Action{Kind: "rmdir", From: d}) + res.DirsRemoved++ + d = filepath.Dir(d) + } +} diff --git a/pkg/organize/duplicates.go b/pkg/organize/duplicates.go new file mode 100644 index 000000000..b72127c50 --- /dev/null +++ b/pkg/organize/duplicates.go @@ -0,0 +1,452 @@ +package organize + +import ( + "encoding/json" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "sync" + "time" + + "github.com/jinzhu/gorm" + + "github.com/xbapps/xbvr/pkg/common" + "github.com/xbapps/xbvr/pkg/models" +) + +// DuplicateDismissal marks a scene's duplicates as ignored (legacy, per-scene). +type DuplicateDismissal struct { + SceneID uint `gorm:"primary_key" json:"sceneId"` +} + +// DuplicateFileIgnore hides a single file from the duplicate list. +type DuplicateFileIgnore struct { + FileID uint `gorm:"primary_key" json:"fileId"` +} + +// DuplicateReport caches the analysis for one scene (Report is a JSON DupGroup). +type DuplicateReport struct { + SceneID uint `gorm:"primary_key" json:"sceneId"` + Report string `gorm:"type:text" json:"-"` + AnalyzedAt time.Time `json:"analyzedAt"` +} + +// DupFile describes one file in a duplicate group. +type DupFile struct { + FileID uint `json:"fileId"` + Path string `json:"path"` + Filename string `json:"filename"` + Height int `json:"height"` + Width int `json:"width"` + Bitrate int `json:"bitrate"` + Size int64 `json:"size"` + Duration float64 `json:"duration"` + Projection string `json:"projection"` + PSNR float64 `json:"psnr"` // vs the reference file (0 for the reference itself) + Suggest string `json:"suggest"` // "keep" | "delete" | "review" + Ignored bool `json:"ignored"` +} + +// DupGroup is the analysis result for one scene with multiple distinct video files. +type DupGroup struct { + SceneID uint `json:"sceneId"` + Title string `json:"title"` + Site string `json:"site"` + Files []DupFile `json:"files"` + KeepFileID uint `json:"keepFileId"` + DurationSpread float64 `json:"durationSpread"` + DurationMismatch bool `json:"durationMismatch"` + Status string `json:"status"` + Detail string `json:"detail"` + AnalyzedAt time.Time `json:"analyzedAt"` +} + +const psnrSampleSeconds = 2 + +var ( + dupMu sync.Mutex + dupRunning bool + dupDone int + dupTotal int + psnrLineRe = regexp.MustCompile(`average:([0-9.]+|inf|nan)`) +) + +func ffmpegBin() string { + p := filepath.Join(common.BinDir, "ffmpeg") + if _, err := os.Stat(p); err == nil { + return p + } + return "ffmpeg" +} + +// StartDupAnalysis runs the (expensive) duplicate analysis in the background. +func StartDupAnalysis(force bool) bool { + // Shared with the organize runner so the two can't touch the same files at once. + if !busy.TryLock() { + return false + } + dupMu.Lock() + dupRunning = true + dupDone, dupTotal = 0, 0 + dupMu.Unlock() + + go func() { + defer busy.Unlock() + db, err := models.GetDB() + if err == nil { + analyzeDuplicates(db, force) + db.Close() + } + dupMu.Lock() + dupRunning = false + dupMu.Unlock() + }() + return true +} + +// DupStatus reports progress. +func DupStatus() (running bool, done, total int) { + dupMu.Lock() + defer dupMu.Unlock() + return dupRunning, dupDone, dupTotal +} + +func analyzeDuplicates(db *gorm.DB, force bool) { + dismissed := map[uint]bool{} + var dis []DuplicateDismissal + db.Find(&dis) + for _, d := range dis { + dismissed[d.SceneID] = true + } + + var ids []uint + db.Model(&models.File{}).Where("type = ? AND scene_id > 0", "video"). + Group("scene_id").Having("count(*) > 1").Pluck("scene_id", &ids) + + dupMu.Lock() + dupTotal = len(ids) + dupMu.Unlock() + + for _, sid := range ids { + dupMu.Lock() + dupDone++ + dupMu.Unlock() + if dismissed[sid] { + continue + } + if !force { + var existing DuplicateReport + if db.Where("scene_id = ?", sid).First(&existing).Error == nil { + continue // already analysed + } + } + var scene models.Scene + if db.Preload("Files").First(&scene, sid).Error != nil { + continue + } + g := analyzeScene(&scene) + if g == nil { + // no longer a real duplicate group; drop any stale report + db.Where("scene_id = ?", sid).Delete(&DuplicateReport{}) + continue + } + data, _ := json.Marshal(g) + db.Save(&DuplicateReport{SceneID: sid, Report: string(data), AnalyzedAt: g.AnalyzedAt}) + } + common.Log.Infof("organize: duplicate analysis complete (%d scenes examined)", len(ids)) +} + +// analyzeScene builds a DupGroup for a scene, or nil if it has < 2 distinct files. +func analyzeScene(scene *models.Scene) *DupGroup { + // distinct video files on disk, keyed by (size, os_hash) to skip exact dups, + // and by inode to skip hard links. + seenKey := map[string]bool{} + seenIno := map[[2]uint64]bool{} + var files []DupFile + for i := range scene.Files { + f := &scene.Files[i] + if f.Type != "video" { + continue + } + fi, err := os.Stat(f.GetPath()) + if err != nil { + continue + } + ino := inodeKey(fi) + if seenIno[ino] { + continue + } + key := strconv.FormatInt(fi.Size(), 10) + ":" + f.OsHash + if f.OsHash != "" && seenKey[key] { + continue + } + seenIno[ino] = true + seenKey[key] = true + files = append(files, DupFile{ + FileID: f.ID, Path: f.Path, Filename: f.Filename, + Height: f.VideoHeight, Width: f.VideoWidth, Bitrate: f.VideoBitRate, + Size: fi.Size(), Duration: f.VideoDuration, Projection: f.VideoProjection, + }) + } + if len(files) < 2 { + return nil + } + + // reference = best (height, bitrate, size) + ref := 0 + for i := range files { + a, b := files[i], files[ref] + if a.Height > b.Height || (a.Height == b.Height && (a.Bitrate > b.Bitrate || (a.Bitrate == b.Bitrate && a.Size > b.Size))) { + ref = i + } + } + + // duration spread + minD, maxD := files[0].Duration, files[0].Duration + for _, f := range files { + if f.Duration < minD { + minD = f.Duration + } + if f.Duration > maxD { + maxD = f.Duration + } + } + spread := maxD - minD + mismatch := maxD > 0 && spread > maxFloat(2.0, 0.03*maxD) + + g := &DupGroup{ + SceneID: scene.ID, Title: scene.Title, Site: scene.Site, + KeepFileID: files[ref].FileID, DurationSpread: spread, + DurationMismatch: mismatch, AnalyzedAt: time.Now(), + } + + refPath := filepath.Join(files[ref].Path, files[ref].Filename) + worst := 999.0 + for i := range files { + if i == ref { + files[i].PSNR = 0 + files[i].Suggest = "keep" + continue + } + if mismatch { + files[i].Suggest = "review" + continue + } + p := filepath.Join(files[i].Path, files[i].Filename) + psnr := psnrBetween(refPath, p, minD) + files[i].PSNR = round2(psnr) + if psnr < worst { + worst = psnr + } + if psnr < 20 { + files[i].Suggest = "review" + } else { + files[i].Suggest = "delete" + } + } + g.Files = files + + switch { + case mismatch: + g.Status = "duration-mismatch" + g.Detail = "Files differ in length (" + dur(minD) + " vs " + dur(maxD) + ") — one may be mis-assigned to this scene. Review before deleting." + case worst < 20: + g.Status = "low-psnr" + g.Detail = "Low PSNR — the files may be different content despite sharing a scene. Review." + case worst >= 40: + g.Status = "identical" + g.Detail = "Near-identical content; keep the highest-quality file, the rest are safe to delete." + default: + g.Status = "same-content" + g.Detail = "Same content at differing quality; keep the highest resolution/bitrate." + } + return g +} + +// psnrBetween returns the average PSNR of b against reference a, sampled at a few +// aligned points and scaled to a common size. Returns 99 for identical (inf). +func psnrBetween(a, b string, minDur float64) float64 { + if minDur <= 0 { + minDur = 30 + } + fracs := []float64{0.2, 0.5, 0.8} + var vals []float64 + for _, fr := range fracs { + t := int(minDur * fr) + if v, ok := psnrSample(a, b, t); ok { + vals = append(vals, v) + } + } + if len(vals) == 0 { + return 0 + } + sum := 0.0 + for _, v := range vals { + sum += v + } + return sum / float64(len(vals)) +} + +func psnrSample(a, b string, t int) (float64, bool) { + const scale = "scale=1280:720:force_original_aspect_ratio=decrease,pad=1280:720:(ow-iw)/2:(oh-ih)/2,setsar=1,fps=5" + cmd := exec.Command(ffmpegBin(), "-hide_banner", + "-ss", strconv.Itoa(t), "-t", strconv.Itoa(psnrSampleSeconds), "-i", a, + "-ss", strconv.Itoa(t), "-t", strconv.Itoa(psnrSampleSeconds), "-i", b, + "-lavfi", "[0:v]"+scale+"[r];[1:v]"+scale+"[d];[d][r]psnr", "-f", "null", "-") + out, _ := cmd.CombinedOutput() + m := psnrLineRe.FindSubmatch(out) + if m == nil { + return 0, false + } + s := string(m[1]) + if s == "inf" { + return 99, true + } + if s == "nan" { + return 0, false + } + v, err := strconv.ParseFloat(s, 64) + if err != nil { + return 0, false + } + return v, true +} + +// ListDupGroups returns the cached duplicate groups. Ignored files are hidden unless +// showIgnored is set; a group with fewer than two visible files is omitted. +func ListDupGroups(db *gorm.DB, showIgnored bool) []DupGroup { + ignored := map[uint]bool{} + var igs []DuplicateFileIgnore + db.Find(&igs) + for _, x := range igs { + ignored[x.FileID] = true + } + var reps []DuplicateReport + db.Order("analyzed_at desc").Find(&reps) + out := []DupGroup{} + for _, r := range reps { + var g DupGroup + if json.Unmarshal([]byte(r.Report), &g) != nil { + continue + } + var files []DupFile + nonIgnored := 0 + for _, f := range g.Files { + f.Ignored = ignored[f.FileID] + if f.Ignored && !showIgnored { + continue + } + if !f.Ignored { + nonIgnored++ + } + files = append(files, f) + } + g.Files = files + if !showIgnored && nonIgnored < 2 { + continue // collapsed by ignores + } + if len(files) == 0 { + continue + } + out = append(out, g) + } + return out +} + +// IgnoreFile / UnignoreFile hide or restore a single file in the duplicate list. +func IgnoreFile(db *gorm.DB, fileID uint) { + db.Save(&DuplicateFileIgnore{FileID: fileID}) +} +func UnignoreFile(db *gorm.DB, fileID uint) { + db.Delete(&DuplicateFileIgnore{}, fileID) +} + +// DeleteFiles removes the given files from disk and DB, refreshing scene status. +func DeleteFiles(db *gorm.DB, ids []uint) int { + n := 0 + scenes := map[uint]bool{} + for _, id := range ids { + var f models.File + if db.First(&f, id).Error != nil { + continue + } + if err := os.Remove(f.GetPath()); err != nil && !os.IsNotExist(err) { + common.Log.Warnf("organize: delete %s failed: %v", f.GetPath(), err) + continue + } + if f.SceneID != 0 { + scenes[f.SceneID] = true + } + db.Delete(&models.File{}, id) + n++ + } + for sid := range scenes { + var s models.Scene + if db.First(&s, sid).Error == nil { + s.UpdateStatus() + } + db.Where("scene_id = ?", sid).Delete(&DuplicateReport{}) + } + common.Log.Infof("organize: deleted %d duplicate file(s)", n) + return n +} + +// DisassociateFiles detaches files from their scene (xbvr "unmatch"): scene_id -> 0, +// removes the filename from the scene's FilenamesArr so it won't be auto-rematched, +// records the action and refreshes scene status. The file stays on disk. +func DisassociateFiles(db *gorm.DB, ids []uint) int { + n := 0 + scenes := map[uint]bool{} + for _, id := range ids { + var f models.File + if db.Where(&models.File{ID: id}).First(&f).Error != nil { + continue + } + sceneID := f.SceneID + if sceneID == 0 { + continue + } + f.SceneID = 0 + f.Save() + + var scene models.Scene + if scene.GetIfExistByPK(sceneID) == nil { + var arr []string + if json.Unmarshal([]byte(scene.FilenamesArr), &arr) == nil { + na := []string{} + for _, fn := range arr { + if fn != f.Filename { + na = append(na, fn) + } + } + if b, err := json.Marshal(na); err == nil { + scene.FilenamesArr = string(b) + } + } + models.AddAction(scene.SceneID, "unmatch", "filenames_arr", scene.FilenamesArr) + scene.UpdateStatus() + scenes[sceneID] = true + } + n++ + } + for sid := range scenes { + db.Where("scene_id = ?", sid).Delete(&DuplicateReport{}) + } + common.Log.Infof("organize: disassociated %d file(s) from their scenes", n) + return n +} + +func maxFloat(a, b float64) float64 { + if a > b { + return a + } + return b +} +func round2(v float64) float64 { return float64(int(v*100+0.5)) / 100 } +func dur(sec float64) string { + m := int(sec) / 60 + s := int(sec) % 60 + return strconv.Itoa(m) + "m" + strconv.Itoa(s) + "s" +} diff --git a/pkg/organize/organize.go b/pkg/organize/organize.go new file mode 100644 index 000000000..e7c1bbf05 --- /dev/null +++ b/pkg/organize/organize.go @@ -0,0 +1,317 @@ +// Package organize reorganises xbvr-managed VR videos into a canonical folder layout +// and keeps the files table in sync. It is a Go port of the standalone organize_vr.py +// tool, running inside xbvr so it can update the DB (and paths are already the +// container's volume paths, so no host mapping is needed). +// +// Target layout: ASeries/{Studio}/{Studio}.{YY}.{MM}.{DD}.{Cast}.{Title}.XXX.{FOV}.{Height}p/ +package organize + +import ( + "crypto/md5" + "encoding/hex" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "sync/atomic" + "syscall" + "time" + + "github.com/xbapps/xbvr/pkg/models" +) + +const maxCast = 4 + +var videoExts = map[string]bool{ + ".mp4": true, ".mkv": true, ".wmv": true, ".avi": true, + ".mov": true, ".m4v": true, ".ts": true, ".webm": true, +} + +var sidecarExts = map[string]bool{ + ".srt": true, ".ass": true, ".ssa": true, ".vtt": true, ".sub": true, ".smi": true, ".idx": true, + ".funscript": true, ".hsp": true, + ".nfo": true, ".json": true, ".txt": true, + ".jpg": true, ".jpeg": true, ".png": true, ".webp": true, ".gif": true, ".bmp": true, +} + +var fovMap = map[string]string{ + "180_sbs": "VR180", "360_tb": "VR360", "mkx200": "MKX200", "mkx220": "MKX220", + "vrca220": "VRCA220", "fisheye190": "FISHEYE190", "fisheye": "FISHEYE", "rf52": "RF52", +} + +// Options controls a run. +type Options struct { + DryRun bool `json:"dryRun"` + Limit int `json:"limit"` // 0 = all; else consider at most N video files + Dedup bool `json:"dedup"` // delete byte-identical copies + DeferDups bool `json:"deferDups"` // skip scenes with possible dups + IncomingDir string `json:"incomingDir"` // staging dir; "" disables + IncomingMinAge int `json:"incomingMinAge"` // days + TopFolder string `json:"topFolder"` // wrapper folder under the storage-folder root; "" = studio folders directly in the root + CastGender string `json:"castGender"` // folder-name cast preference: "any", "female", or "male" + SymlinkByActor bool `json:"symlinkByActor"` // also symlink each scene dir into per-actor folders + ActorFolder string `json:"actorFolder"` // parent folder for the per-actor symlink dirs +} + +// Action is one planned/performed operation (for the UI preview). +type Action struct { + SceneID uint `json:"sceneId"` + Kind string `json:"kind"` // move|rename|dedup-delete|dedup-maybe|hardlink-unlink|sidecar|symlink|symlink-prune|rmdir + From string `json:"from"` + To string `json:"to,omitempty"` + Note string `json:"note,omitempty"` +} + +// Result summarises a run. +type Result struct { + DryRun bool `json:"dryRun"` + ScenesActed int `json:"scenesActed"` + FilesMoved int `json:"filesMoved"` + FilesRenamed int `json:"filesRenamed"` + Dedups int `json:"identicalCopiesDeleted"` + Hardlinks int `json:"hardlinksUnlinked"` + Sidecars int `json:"sidecarsMoved"` + DirsRemoved int `json:"emptyDirsRemoved"` + Deferred int `json:"scenesDeferred"` + Held int `json:"filesHeldRecent"` + Merged int `json:"mergedDuplicateScenes"` + Symlinks int `json:"actorSymlinksCreated"` + SymlinksPruned int `json:"actorSymlinksPruned"` + BytesReclaimed int64 `json:"bytesReclaimed"` + Actions []Action `json:"actions"` +} + +// ---- sanitisation ---- + +var reParen = regexp.MustCompile(`\s*\([^)]*\)\s*$`) +var reWS = regexp.MustCompile(`\s+`) +var reNonWord = regexp.MustCompile(`[^\w]`) +var reNonWordDot = regexp.MustCompile(`[^\w\.]`) +var reDoubleDot = regexp.MustCompile(`\.\..+$`) + +func sanitizeSite(site string) string { + site = reParen.ReplaceAllString(site, "") + site = reWS.ReplaceAllString(site, "") + return reNonWord.ReplaceAllString(site, "") +} + +var reWordSplit = regexp.MustCompile(`[^A-Za-z0-9]+`) + +// camelActor renders a performer name as a CamelCase folder name (e.g. "Shalina +// Devine" -> "ShalinaDevine"), capitalising each word and stripping separators. +func camelActor(name string) string { + name = reParen.ReplaceAllString(name, "") + var b strings.Builder + for _, part := range reWordSplit.Split(name, -1) { + if part == "" { + continue + } + r := []rune(part) + b.WriteString(strings.ToUpper(string(r[0]))) + if len(r) > 1 { + b.WriteString(string(r[1:])) + } + } + return b.String() +} + +func sanitizeToken(text string) string { + if text == "" { + return "" + } + text = reWS.ReplaceAllString(text, ".") + text = reNonWordDot.ReplaceAllString(text, "") + text = reDoubleDot.ReplaceAllString(text, "") + return strings.Trim(text, ".") +} + +func fovToken(projection string) string { + if projection == "" { + return "VR180" + } + if v, ok := fovMap[strings.ToLower(strings.TrimSpace(projection))]; ok { + return v + } + cleaned := reNonWord.ReplaceAllString(strings.ToUpper(projection), "") + if cleaned == "" { + return "VR180" + } + return cleaned +} + +func parseReleaseDate(s models.Scene) (time.Time, bool) { + if s.ReleaseDateText != "" && len(s.ReleaseDateText) >= 10 { + if t, err := time.Parse("2006-01-02", s.ReleaseDateText[:10]); err == nil { + return t, true + } + } + if !s.ReleaseDate.IsZero() { + return s.ReleaseDate, true + } + return time.Time{}, false +} + +// selectCast picks the performers used for naming: drop "aka:" alias records (unless +// they're all there is), then select by the configured gender preference. With a +// preference of "female" or "male", performers of that gender are used (falling back to +// unknown-gender performers when none match); "any" (or empty) keeps every performer +// regardless of gender. Names are returned raw (unsanitised), capped at maxCast. +func selectCast(cast []models.Actor, pref string) []string { + // Match the original tool: consider performers in alphabetical order by name so + // the folder cast order is stable (and doesn't churn already-correct folders). + sorted := make([]models.Actor, len(cast)) + copy(sorted, cast) + sort.Slice(sorted, func(i, j int) bool { return sorted[i].Name < sorted[j].Name }) + + real := sorted[:0:0] + for _, a := range sorted { + if !strings.HasPrefix(strings.ToLower(a.Name), "aka:") { + real = append(real, a) + } + } + pool := real + if len(pool) == 0 { + pool = sorted + } + pref = strings.ToLower(strings.TrimSpace(pref)) + var preferred, unknowns []string + for _, a := range pool { + g := strings.ToLower(strings.TrimSpace(a.Gender)) + switch pref { + case "female", "male": + switch g { + case pref: + preferred = append(preferred, a.Name) + case "female", "male", "non_binary": + // a specified gender that isn't the preferred one; excluded + default: + unknowns = append(unknowns, a.Name) + } + default: + // no preference: keep every performer + preferred = append(preferred, a.Name) + } + } + chosen := preferred + if len(chosen) == 0 { + chosen = unknowns + } + if len(chosen) > maxCast { + chosen = chosen[:maxCast] + } + return chosen +} + +// castNames returns the selected performers as sanitised folder-name tokens. +func castNames(cast []models.Actor, pref string) []string { + chosen := selectCast(cast, pref) + out := make([]string, 0, len(chosen)) + for _, n := range chosen { + if tok := sanitizeToken(n); tok != "" { + out = append(out, tok) + } + } + return out +} + +func buildDirName(site string, dt time.Time, cast []string, title, fov string, height int) (studio, dir string) { + studio = sanitizeSite(site) + segs := []string{studio, dt.Format("06.01.02")} + if c := strings.Join(cast, "."); c != "" { + segs = append(segs, c) + } + segs = append(segs, title, "XXX", fov, strconv.Itoa(height)+"p") + var kept []string + for _, s := range segs { + if s != "" { + kept = append(kept, s) + } + } + return studio, strings.Join(kept, ".") +} + +// ---- fs helpers ---- + +func splitExt(name string) (base, ext string) { + ext = filepath.Ext(name) + return name[:len(name)-len(ext)], ext +} + +func resolveName(used map[string]bool, name string, height int) string { + if !used[name] { + return name + } + base, ext := splitExt(name) + cand := base + "." + strconv.Itoa(height) + "p" + ext + if !used[cand] { + return cand + } + for i := 2; ; i++ { + cand = base + "." + strconv.Itoa(height) + "p." + strconv.Itoa(i) + ext + if !used[cand] { + return cand + } + } +} + +var inodeFallback uint64 + +func inodeKey(fi os.FileInfo) [2]uint64 { + if st, ok := fi.Sys().(*syscall.Stat_t); ok { + return [2]uint64{uint64(st.Dev), uint64(st.Ino)} + } + // No inode info: give each file a unique key so distinct files are never mistaken + // for hard links of one another (which would delete all but one). + return [2]uint64{^uint64(0), atomic.AddUint64(&inodeFallback, 1)} +} + +func fileMD5(path string) (string, bool) { + f, err := os.Open(path) + if err != nil { + return "", false + } + defer f.Close() + h := md5.New() + if _, err := io.Copy(h, f); err != nil { + return "", false + } + return hex.EncodeToString(h.Sum(nil)), true +} + +func moveFile(src, dst string) error { + // Never overwrite an existing destination (defense-in-depth against a TOCTOU race + // between planning and applying). + if src != dst { + if _, err := os.Lstat(dst); err == nil { + return fmt.Errorf("destination already exists: %s", dst) + } + } + if err := os.Rename(src, dst); err == nil { + return nil + } + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + out, err := os.Create(dst) + if err != nil { + return err + } + if _, err := io.Copy(out, in); err != nil { + out.Close() + return err + } + if err := out.Close(); err != nil { + return err + } + return os.Remove(src) +} + +func isUnder(path, dir string) bool { + return dir != "" && (path == dir || strings.HasPrefix(path, dir+"/")) +} diff --git a/pkg/organize/run.go b/pkg/organize/run.go new file mode 100644 index 000000000..49e528781 --- /dev/null +++ b/pkg/organize/run.go @@ -0,0 +1,570 @@ +package organize + +import ( + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/jinzhu/gorm" + + "github.com/xbapps/xbvr/pkg/common" + "github.com/xbapps/xbvr/pkg/models" +) + +type moveOp struct { + file *models.File + src string + dst string + name string + isMove bool +} +type dupOp struct { + file *models.File + src string + identicalTo string + size int64 + maybe bool // dry-run only: OsHash match, md5 not yet confirmed +} +type scenePlan struct { + sceneID uint + targetDir string + volumeRoot string + videoCount int + actionable bool + moves []moveOp + hlUnlink []*models.File + contentDup []dupOp + sidecars [][2]string // {src, dst} + symlinks [][2]string // {linkPath, target} per-actor symlinks to targetDir + rmdirs map[string]bool +} + +// Run computes (and, unless DryRun, performs) the reorganisation and returns a Result. +func Run(db *gorm.DB, opts Options) *Result { + res := &Result{DryRun: opts.DryRun} + now := time.Now() + cutoff := now.Add(-time.Duration(opts.IncomingMinAge) * 24 * time.Hour).Unix() + + // Storage folders (local volumes) are the move boundaries: a scene is only ever + // reorganised within the folder its files already live in, never across folders. + volRoots := localVolumeRoots(db) + if len(volRoots) == 0 { + common.Log.Warn("organize: no local storage folders configured; nothing to do") + return res + } + + // dir (db path) -> set of scene ids with a video file there (immutable snapshot) + dirScenes := map[string]map[uint]bool{} + var frows []models.File + db.Model(&models.File{}).Select("path, scene_id").Where("type = ? AND scene_id > 0", "video").Find(&frows) + for _, f := range frows { + if dirScenes[f.Path] == nil { + dirScenes[f.Path] = map[uint]bool{} + } + dirScenes[f.Path][f.SceneID] = true + } + + var sceneIDs []uint + db.Model(&models.File{}).Where("type = ? AND scene_id > 0", "video").Order("scene_id").Pluck("DISTINCT scene_id", &sceneIDs) + + targetOwner := map[string]uint{} + removedPaths := map[string]bool{} + dryRemovedDirs := map[string]bool{} + considered := 0 + + for _, sid := range sceneIDs { + if opts.Limit > 0 && considered >= opts.Limit { + break + } + var scene models.Scene + if err := db.Preload("Cast").Preload("Files").First(&scene, sid).Error; err != nil { + continue + } + if scene.DeletedAt != nil { + continue + } + plan, deferred := processScene(db, &scene, opts, volRoots, cutoff, dirScenes, res) + if deferred { + res.Deferred++ + continue + } + if plan == nil || !plan.actionable { + continue + } + if _, ok := targetOwner[plan.targetDir]; ok { + res.Merged++ + } else { + targetOwner[plan.targetDir] = sid + } + applyPlan(db, plan, opts, res, removedPaths, dryRemovedDirs) + res.ScenesActed++ + considered += plan.videoCount + } + if opts.SymlinkByActor { + pruneDanglingActorSymlinks(volRoots, opts, res) + } + common.Log.Infof("organize [dry=%v]: %d scenes acted, %d moved, %d renamed, %d dedup(-%0.1fGB), %d hardlinks, %d sidecars, %d symlinks (%d pruned), %d dirs removed, %d deferred, %d held", + opts.DryRun, res.ScenesActed, res.FilesMoved, res.FilesRenamed, res.Dedups, + float64(res.BytesReclaimed)/1e9, res.Hardlinks, res.Sidecars, res.Symlinks, res.SymlinksPruned, res.DirsRemoved, res.Deferred, res.Held) + return res +} + +// pruneDanglingActorSymlinks removes per-actor symlinks whose scene directory no longer +// exists (e.g. after a scene was renamed) and drops emptied actor folders. +func pruneDanglingActorSymlinks(volRoots []string, opts Options, res *Result) { + for _, vroot := range volRoots { + actorRoot := filepath.Join(vroot, opts.TopFolder, opts.ActorFolder) + actors, err := os.ReadDir(actorRoot) + if err != nil { + continue + } + for _, a := range actors { + if !a.IsDir() { + continue + } + adir := filepath.Join(actorRoot, a.Name()) + links, err := os.ReadDir(adir) + if err != nil { + continue + } + live := 0 + for _, l := range links { + link := filepath.Join(adir, l.Name()) + if l.Type()&os.ModeSymlink == 0 { + live++ + continue + } + if _, err := os.Stat(link); os.IsNotExist(err) { + if !opts.DryRun { + if err := os.Remove(link); err != nil { + live++ + continue + } + } + res.Actions = append(res.Actions, Action{Kind: "symlink-prune", From: link}) + res.SymlinksPruned++ + continue + } + live++ + } + if live == 0 && !opts.DryRun { + os.Remove(adir) // emptied actor folder + } + } + } +} + +// localVolumeRoots returns the enabled local storage-folder roots, longest first so a +// path is matched to the deepest containing volume. +func localVolumeRoots(db *gorm.DB) []string { + var vols []models.Volume + db.Where("type = ? AND is_enabled = ?", "local", true).Find(&vols) + roots := make([]string, 0, len(vols)) + for _, v := range vols { + if r := strings.TrimRight(v.Path, "/"); r != "" { + roots = append(roots, r) + } + } + sort.Slice(roots, func(i, j int) bool { return len(roots[i]) > len(roots[j]) }) + return roots +} + +// volumeOf returns the storage-folder root that contains path, or "" if none does. +func volumeOf(path string, roots []string) string { + for _, r := range roots { + if path == r || strings.HasPrefix(path, r+"/") { + return r + } + } + return "" +} + +func lookupOsHash(db *gorm.DB, path string) string { + var f models.File + if db.Select("os_hash").Where("path = ? AND filename = ?", filepath.Dir(path), filepath.Base(path)).First(&f).Error == nil { + return f.OsHash + } + return "" +} + +// processScene builds the plan for one scene. Returns (nil, true) when the whole +// scene is deferred (possible duplicates + DeferDups). +func processScene(db *gorm.DB, scene *models.Scene, opts Options, volRoots []string, cutoff int64, dirScenes map[string]map[uint]bool, res *Result) (*scenePlan, bool) { + dt, ok := parseReleaseDate(*scene) + title := sanitizeToken(scene.Title) + if scene.Site == "" || !ok || title == "" { + return nil, false + } + + type exFile struct { + f *models.File + disk string + height int + ino [2]uint64 + size int64 + } + var existing []exFile + for i := range scene.Files { + f := &scene.Files[i] + if f.Type != "video" { + continue + } + disk := f.GetPath() + fi, err := os.Stat(disk) + if err != nil { + continue + } + // Staging: hold files still sitting in a storage folder's incoming area that are + // too recently modified to judge. IncomingDir is relative to each storage folder. + if opts.IncomingDir != "" { + if fv := volumeOf(disk, volRoots); fv != "" && + isUnder(disk, filepath.Join(fv, opts.IncomingDir)) && fi.ModTime().Unix() > cutoff { + res.Held++ + continue + } + } + existing = append(existing, exFile{f, disk, f.VideoHeight, inodeKey(fi), fi.Size()}) + } + if len(existing) == 0 { + return nil, false + } + + // representative = max(height, size, -id) + pickRep := func(xs []exFile) int { + rep := 0 + for i := range xs { + a, b := xs[i], xs[rep] + if a.height > b.height || (a.height == b.height && (a.size > b.size || (a.size == b.size && a.f.ID < b.f.ID))) { + rep = i + } + } + return rep + } + + // Organise within the storage folder holding the best copy; only files already in + // that folder are touched, so nothing is ever moved across storage folders. + vroot := volumeOf(existing[pickRep(existing)].disk, volRoots) + if vroot == "" { + return nil, false + } + kept := existing[:0:0] + for _, e := range existing { + if volumeOf(e.disk, volRoots) == vroot { + kept = append(kept, e) + } + } + existing = kept + if len(existing) == 0 { + return nil, false + } + + rep := pickRep(existing) + repHeight := existing[rep].height + for _, e := range existing { + if e.height > repHeight { + repHeight = e.height + } + } + fov := fovToken(existing[rep].f.VideoProjection) + cast := castNames(scene.Cast, opts.CastGender) + studio, dirName := buildDirName(scene.Site, dt, cast, title, fov, repHeight) + if studio == "" { + return nil, false + } + + plan := &scenePlan{sceneID: scene.ID, videoCount: len(existing), volumeRoot: vroot, rmdirs: map[string]bool{}} + // TopFolder is an optional wrapper under the storage-folder root ("" -> studio + // folders go directly in the root). filepath.Join drops the empty element. + plan.targetDir = filepath.Join(vroot, opts.TopFolder, studio, dirName) + + // group by inode (hard links within scene) + byInode := map[[2]uint64][]int{} + for i := range existing { + byInode[existing[i].ino] = append(byInode[existing[i].ino], i) + } + var keepers []int + for _, grp := range byInode { + keep := grp[0] + for _, idx := range grp { + if idx == rep { + keep = rep + } + } + if keep != rep { + // pick max within group + for _, idx := range grp { + a, b := existing[idx], existing[keep] + if a.height > b.height || (a.height == b.height && a.size > b.size) { + keep = idx + } + } + } + keepers = append(keepers, keep) + for _, idx := range grp { + if idx != keep { + plan.hlUnlink = append(plan.hlUnlink, existing[idx].f) + } + } + } + // order keepers: representative first, then by descending resolution/size + sort.Slice(keepers, func(i, j int) bool { + if (keepers[i] == rep) != (keepers[j] == rep) { + return keepers[i] == rep + } + a, b := existing[keepers[i]], existing[keepers[j]] + if a.height != b.height { + return a.height > b.height + } + return a.size > b.size + }) + + // content dedup + type cand struct { + osHash string + path string + ino [2]uint64 + } + sizeIdx := map[int64][]cand{} + keeperDisks := map[string]bool{} + for _, k := range keepers { + keeperDisks[existing[k].disk] = true + } + if opts.Dedup { + if entries, err := os.ReadDir(plan.targetDir); err == nil { + for _, e := range entries { + if e.IsDir() { + continue + } + p := filepath.Join(plan.targetDir, e.Name()) + if keeperDisks[p] { + continue + } + if fi, err := e.Info(); err == nil { + sizeIdx[fi.Size()] = append(sizeIdx[fi.Size()], cand{lookupOsHash(db, p), p, inodeKey(fi)}) + } + } + } + } + var survivors []int + for _, k := range keepers { + e := existing[k] + var hardlinkOf, dupOf string + var dupMaybe bool + cands := sizeIdx[e.size] + if opts.Dedup { + for _, c := range cands { + if c.ino == e.ino { + hardlinkOf = c.path + break + } + } + if hardlinkOf == "" { + for _, c := range cands { + if e.f.OsHash != "" && c.osHash != "" && e.f.OsHash == c.osHash { + if opts.DeferDups { + return nil, true + } + if opts.DryRun { + // Preview only: OsHash matched but md5 isn't verified here, so + // this is a candidate, not a confirmed delete. + dupOf = c.path + dupMaybe = true + break + } + if m1, ok1 := fileMD5(e.disk); ok1 { + if m2, ok2 := fileMD5(c.path); ok2 && m1 == m2 { + dupOf = c.path + break + } + } + } + } + } + } + switch { + case hardlinkOf != "": + plan.hlUnlink = append(plan.hlUnlink, e.f) + plan.actionable = true + case dupOf != "": + plan.contentDup = append(plan.contentDup, dupOp{e.f, e.disk, dupOf, e.size, dupMaybe}) + plan.actionable = true + default: + survivors = append(survivors, k) + sizeIdx[e.size] = append(sizeIdx[e.size], cand{e.f.OsHash, e.disk, e.ino}) + } + } + keepers = survivors + for _, d := range plan.contentDup { + if !d.maybe { + plan.rmdirs[filepath.Dir(d.src)] = true + } + } + + // name assignment + sidecars + sceneDiskPaths := map[string]bool{} + for _, k := range keepers { + sceneDiskPaths[existing[k].disk] = true + } + for _, f := range plan.hlUnlink { + sceneDiskPaths[f.GetPath()] = true + } + // gather sidecars per source dir + type scDir struct { + foreign bool + entries []string + } + sidecarSrc := map[string]*scDir{} + for _, k := range keepers { + dir := filepath.Dir(existing[k].disk) + if _, done := sidecarSrc[dir]; done { + continue + } + sd := &scDir{} + for s := range dirScenes[dir] { + if s != scene.ID { + sd.foreign = true + } + } + if entries, err := os.ReadDir(dir); err == nil { + for _, e := range entries { + if e.IsDir() { + continue + } + p := filepath.Join(dir, e.Name()) + if sceneDiskPaths[p] { + continue + } + ext := lowerExt(e.Name()) + if videoExts[ext] { + sd.foreign = true + continue + } + if !sidecarExts[ext] { + continue + } + sd.entries = append(sd.entries, p) + } + } + sidecarSrc[dir] = sd + } + + // reserve foreign names in target + placedOrCand := map[string]bool{} + for _, k := range keepers { + placedOrCand[existing[k].disk] = true + } + for _, sd := range sidecarSrc { + for _, p := range sd.entries { + placedOrCand[p] = true + } + } + used := map[string]bool{} + if entries, err := os.ReadDir(plan.targetDir); err == nil { + for _, e := range entries { + if e.IsDir() { + continue + } + p := filepath.Join(plan.targetDir, e.Name()) + if !placedOrCand[p] { + used[e.Name()] = true + } + } + } + assign := func(src, desired string, height int) (string, bool) { + if filepath.Dir(src) == plan.targetDir { + name := filepath.Base(src) + used[name] = true + return name, false + } + name := resolveName(used, desired, height) + used[name] = true + return name, filepath.Join(plan.targetDir, name) != src + } + + stemmap := map[string]map[string]string{} // srcDir -> origStem -> finalStem + for _, k := range keepers { + e := existing[k] + h := e.height + if k == rep { + h = repHeight + } + name, isMove := assign(e.disk, e.f.Filename, h) + if isMove { + plan.actionable = true + } + plan.moves = append(plan.moves, moveOp{e.f, e.disk, filepath.Join(plan.targetDir, name), name, isMove}) + dir := filepath.Dir(e.disk) + if stemmap[dir] == nil { + stemmap[dir] = map[string]string{} + } + origStem, _ := splitExt(e.f.Filename) + finalStem, _ := splitExt(name) + stemmap[dir][origStem] = finalStem + plan.rmdirs[dir] = true + } + if len(plan.hlUnlink) > 0 { + plan.actionable = true + } + for _, f := range plan.hlUnlink { + plan.rmdirs[filepath.Dir(f.GetPath())] = true + } + + for dir, sd := range sidecarSrc { + for _, p := range sd.entries { + name := filepath.Base(p) + var pairedFinal string + for orig, final := range stemmap[dir] { + if name == orig || (len(name) > len(orig) && name[:len(orig)+1] == orig+".") { + pairedFinal = final + name[len(orig):] + break + } + } + var desired string + if pairedFinal != "" { + desired = pairedFinal + } else if !sd.foreign { + desired = name + } else { + continue + } + dstName, isMove := assign(p, desired, repHeight) + if !isMove { + continue + } + plan.sidecars = append(plan.sidecars, [2]string{p, filepath.Join(plan.targetDir, dstName)}) + plan.actionable = true + } + } + + // Per-actor symlinks: link the scene dir into a CamelCase folder per performer. + if opts.SymlinkByActor { + actorRoot := filepath.Join(vroot, opts.TopFolder, opts.ActorFolder) + seen := map[string]bool{} + for _, name := range selectCast(scene.Cast, opts.CastGender) { + camel := camelActor(name) + if camel == "" || seen[camel] { + continue + } + seen[camel] = true + actorDir := filepath.Join(actorRoot, camel) + linkPath := filepath.Join(actorDir, dirName) + if _, err := os.Lstat(linkPath); err == nil { + continue // already linked + } + target, err := filepath.Rel(actorDir, plan.targetDir) + if err != nil { + target = plan.targetDir + } + plan.symlinks = append(plan.symlinks, [2]string{linkPath, target}) + plan.actionable = true + } + } + + return plan, false +} + +func lowerExt(name string) string { + return strings.ToLower(filepath.Ext(name)) +} diff --git a/pkg/organize/runner.go b/pkg/organize/runner.go new file mode 100644 index 000000000..0058175ca --- /dev/null +++ b/pkg/organize/runner.go @@ -0,0 +1,53 @@ +package organize + +import ( + "sync" + + "github.com/xbapps/xbvr/pkg/common" + "github.com/xbapps/xbvr/pkg/models" +) + +var ( + runMu sync.Mutex + running bool + last *Result + + // busy is shared by the organize runner and the duplicate analyzer so the two never + // run at once — they touch the same files on disk. Held for a job's whole lifetime. + busy sync.Mutex +) + +// Start launches a run in the background (preview if opts.DryRun). Returns false if a +// run (or a duplicate analysis) is already in progress. +func Start(opts Options) bool { + if !busy.TryLock() { + return false + } + runMu.Lock() + running = true + runMu.Unlock() + + go func() { + defer busy.Unlock() + var r *Result + db, err := models.GetDB() + if err != nil { + common.Log.Errorf("organize: db open failed: %v", err) + } else { + r = Run(db, opts) + db.Close() + } + runMu.Lock() + last = r + running = false + runMu.Unlock() + }() + return true +} + +// Status reports whether a run is in progress and the most recent result. +func Status() (bool, *Result) { + runMu.Lock() + defer runMu.Unlock() + return running, last +} diff --git a/pkg/server/cron.go b/pkg/server/cron.go index 001fd1964..48cee0110 100644 --- a/pkg/server/cron.go +++ b/pkg/server/cron.go @@ -7,6 +7,7 @@ import ( "github.com/robfig/cron/v3" "github.com/xbapps/xbvr/pkg/api" "github.com/xbapps/xbvr/pkg/config" + "github.com/xbapps/xbvr/pkg/organize" "github.com/xbapps/xbvr/pkg/session" "github.com/xbapps/xbvr/pkg/tasks" ) @@ -18,6 +19,7 @@ var previewTask cron.EntryID var actorScrapeTask cron.EntryID var stashdbScrapeTask cron.EntryID var linkScenesTask cron.EntryID +var organizeTask cron.EntryID func SetupCron() { cronInstance = cron.New() @@ -48,6 +50,10 @@ func SetupCron() { log.Println(fmt.Sprintf("Setup Link Scenes Task %v", formatCronSchedule(config.CronSchedule(config.Config.Cron.LinkScenesSchedule)))) linkScenesTask, _ = cronInstance.AddFunc(formatCronSchedule(config.CronSchedule(config.Config.Cron.LinkScenesSchedule)), linkScenesCron) } + if config.Config.Cron.OrganizeSchedule.Enabled { + log.Println(fmt.Sprintf("Setup Organize Task %v", formatCronSchedule(config.CronSchedule(config.Config.Cron.OrganizeSchedule)))) + organizeTask, _ = cronInstance.AddFunc(formatCronSchedule(config.CronSchedule(config.Config.Cron.OrganizeSchedule)), organizeCron) + } cronInstance.Start() go tasks.CalculateCacheSizes() @@ -70,6 +76,9 @@ func SetupCron() { if config.Config.Cron.LinkScenesSchedule.RunAtStartDelay > 0 { time.AfterFunc(time.Duration(config.Config.Cron.LinkScenesSchedule.RunAtStartDelay)*time.Minute, linkScenesCron) } + if config.Config.Cron.OrganizeSchedule.RunAtStartDelay > 0 { + time.AfterFunc(time.Duration(config.Config.Cron.OrganizeSchedule.RunAtStartDelay)*time.Minute, organizeCron) + } log.Println(fmt.Sprintf("Next Rescrape Task at %v", cronInstance.Entry(rescrapTask).Next)) log.Println(fmt.Sprintf("Next Rescan Task at %v", cronInstance.Entry(rescanTask).Next)) @@ -112,6 +121,25 @@ func linkScenesCron() { log.Println(fmt.Sprintf("Next Link Scenes Task at %v", cronInstance.Entry(rescrapTask).Next)) } +func organizeCron() { + // Scheduled runs are DRY-RUN only: compute + log the plan; the user applies from the UI. + if !session.HasActiveSession() { + c := config.Config.Organize + organize.Start(organize.Options{ + DryRun: true, + Dedup: c.Dedup, + DeferDups: c.DeferDups, + IncomingDir: c.IncomingDir, + IncomingMinAge: c.IncomingMinAge, + TopFolder: c.TopFolder, + CastGender: c.CastGender, + SymlinkByActor: c.SymlinkByActor, + ActorFolder: c.ActorFolder, + }) + } + log.Println(fmt.Sprintf("Next Organize Task at %v", cronInstance.Entry(organizeTask).Next)) +} + var previewGenerateInProgress = false func generatePreviewCron() { diff --git a/pkg/server/server.go b/pkg/server/server.go index 392524274..cc5c85bea 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -95,6 +95,7 @@ func StartServer(version, commit, branch, date string) { restful.Add(api.AkaResource{}.WebService()) restful.Add(api.TagGroupResource{}.WebService()) restful.Add(api.ExternalReference{}.WebService()) + restful.Add(api.OrganizeResource{}.WebService()) restConfig := restfulspec.Config{ WebServices: restful.RegisteredWebServices(), diff --git a/ui/src/store/index.js b/ui/src/store/index.js index 1a7840cd5..d65a12e24 100644 --- a/ui/src/store/index.js +++ b/ui/src/store/index.js @@ -17,6 +17,8 @@ import optionsFunscripts from './optionsFunscripts' import optionsVendor from './optionsVendor' import optionsAdvanced from './optionsAdvanced' import optionsSceneCreate from './optionsSceneCreate' +import optionsOrganize from './optionsOrganize' +import optionsDuplicates from './optionsDuplicates' Vue.use(Vuex) @@ -38,5 +40,7 @@ export default new Vuex.Store({ optionsVendor, optionsAdvanced, optionsSceneCreate, + optionsOrganize, + optionsDuplicates, } }) diff --git a/ui/src/store/optionsDuplicates.js b/ui/src/store/optionsDuplicates.js new file mode 100644 index 000000000..2650d64f0 --- /dev/null +++ b/ui/src/store/optionsDuplicates.js @@ -0,0 +1,50 @@ +import ky from 'ky' + +const state = { + loading: false, + running: false, + showIgnored: false, + done: 0, + total: 0, + groups: [] +} + +const mutations = { + set (state, data) { + state.running = data.running + state.done = data.done || 0 + state.total = data.total || 0 + state.groups = data.groups || [] + }, + setLoading (state, v) { state.loading = v }, + setShowIgnored (state, v) { state.showIgnored = v } +} + +const actions = { + async load ({ state, commit }) { + commit('setLoading', true) + try { + const data = await ky.get('/api/organize/duplicates' + (state.showIgnored ? '?showIgnored=true' : '')).json() + commit('set', data) + } finally { + commit('setLoading', false) + } + }, + async analyze (_ctx, force) { + await ky.post('/api/organize/duplicates/analyze' + (force ? '?force=true' : '')).json() + }, + async ignore (_ctx, fileId) { + await ky.post('/api/organize/duplicates/ignore', { json: { fileId } }).json() + }, + async unignore (_ctx, fileId) { + await ky.post('/api/organize/duplicates/unignore', { json: { fileId } }).json() + }, + async del (_ctx, fileIds) { + await ky.post('/api/organize/duplicates/delete', { json: { fileIds } }).json() + }, + async disassociate (_ctx, fileIds) { + await ky.post('/api/organize/duplicates/disassociate', { json: { fileIds } }).json() + } +} + +export default { namespaced: true, state, mutations, actions } diff --git a/ui/src/store/optionsOrganize.js b/ui/src/store/optionsOrganize.js new file mode 100644 index 000000000..3cebae33e --- /dev/null +++ b/ui/src/store/optionsOrganize.js @@ -0,0 +1,58 @@ +import ky from 'ky' + +const state = { + loading: false, + running: false, + result: null, + config: { + dedup: true, + deferDups: false, + incomingDir: 'Incoming', + incomingMinAge: 30, + topFolder: '', + castGender: 'female', + symlinkByActor: false, + actorFolder: 'ByActor' + } +} + +const mutations = { + setConfig (state, cfg) { state.config = { ...state.config, ...cfg } }, + setField (state, { key, value }) { state.config[key] = value }, + setLoading (state, v) { state.loading = v }, + setStatus (state, { running, result }) { state.running = running; if (result) state.result = result } +} + +const actions = { + async load ({ commit }) { + commit('setLoading', true) + try { + const cfg = await ky.get('/api/organize/config').json() + commit('setConfig', cfg) + const st = await ky.get('/api/organize/status').json() + commit('setStatus', st) + } finally { + commit('setLoading', false) + } + }, + async save ({ state, commit }) { + commit('setLoading', true) + try { + const cfg = await ky.post('/api/organize/config', { json: { ...state.config } }).json() + commit('setConfig', cfg) + } finally { + commit('setLoading', false) + } + }, + async run ({ commit }, { dryRun, limit }) { + await ky.post('/api/organize/run', { json: { dryRun, limit: limit || 0 } }).json() + commit('setStatus', { running: true }) + }, + async pollStatus ({ commit }) { + const st = await ky.get('/api/organize/status').json() + commit('setStatus', st) + return st + } +} + +export default { namespaced: true, state, mutations, actions } diff --git a/ui/src/views/options/Options.vue b/ui/src/views/options/Options.vue index ae591e296..58368ee23 100644 --- a/ui/src/views/options/Options.vue +++ b/ui/src/views/options/Options.vue @@ -10,6 +10,8 @@ + + + + @@ -63,12 +67,14 @@ import InterfaceDLNA from './sections/InterfaceDLNA.vue' import Cache from './sections/Cache.vue' import Previews from './sections/Previews.vue' import Schedules from './sections/Schedules.vue' +import Organize from './sections/Organize.vue' +import Duplicates from './sections/Duplicates.vue' import InterfaceDeoVR from './sections/InterfaceDeoVR.vue' import InterfaceAdvanced from './sections/InterfaceAdvanced.vue' import SceneMatchParams from './overlays/SceneMatchParams.vue' export default { - components: { Storage, SceneDataScrapers, SceneCreate, Funscripts, SceneDataImportExport, InterfaceWeb, InterfaceDLNA, InterfaceDeoVR, Cache, Previews, Schedules, InterfaceAdvanced,SceneMatchParams }, + components: { Storage, SceneDataScrapers, SceneCreate, Funscripts, SceneDataImportExport, InterfaceWeb, InterfaceDLNA, InterfaceDeoVR, Cache, Previews, Schedules, Organize, Duplicates, InterfaceAdvanced,SceneMatchParams }, data: function () { return { active: 'storage' diff --git a/ui/src/views/options/sections/Duplicates.vue b/ui/src/views/options/sections/Duplicates.vue new file mode 100644 index 000000000..0ce385867 --- /dev/null +++ b/ui/src/views/options/sections/Duplicates.vue @@ -0,0 +1,140 @@ + + + diff --git a/ui/src/views/options/sections/Organize.vue b/ui/src/views/options/sections/Organize.vue new file mode 100644 index 000000000..58ebf0bbd --- /dev/null +++ b/ui/src/views/options/sections/Organize.vue @@ -0,0 +1,134 @@ + + + diff --git a/ui/src/views/options/sections/Schedules.vue b/ui/src/views/options/sections/Schedules.vue index 18df4b3c4..5f3bdebc0 100644 --- a/ui/src/views/options/sections/Schedules.vue +++ b/ui/src/views/options/sections/Schedules.vue @@ -11,6 +11,7 @@ +
@@ -250,6 +251,24 @@
{{ delayStartMsg(linkScenesStartDelay) }}
+
+

{{$t("Organize Files")}}

+

Computes the reorganisation plan on a schedule (dry-run only) and logs it — nothing is + moved automatically. Review and Apply from Options → Organize files. Skips while you're + watching. Changes apply after a restart.

+ + Enable schedule + + + +
{{`Run every ${this.organizeHourInterval} hour${this.organizeHourInterval > 1 ? 's': ''}`}}
+ +
+ + +
{{ delayStartMsg(organizeStartDelay) }}
+ +
Save settings @@ -317,8 +336,11 @@ export default { linkScenesHourInterval: 0, linkScenesMinuteStart: 0, lastlinkScenesTimeRange: [0,23], - useLinkScenesTimeRange: false, + useLinkScenesTimeRange: false, linkScenesStartDelay: 0, + organizeEnabled: false, + organizeHourInterval: 24, + organizeStartDelay: 0, timeRange: ['00:00', '01:00', '02:00', '03:00', '04:00', '05:00', '06:00', '07:00', '08:00', '09:00', '10:00', '11:00', '12:00', '13:00', '14:00', '15:00', '16:00', '17:00', '18:00', '19:00', '20:00', '21:00', '22:00', '23:00', '00:00', '01:00', '02:00', '03:00', '04:00', '05:00', '06:00', '07:00', '08:00', '09:00', '10:00', '11:00', @@ -447,7 +469,12 @@ export default { this.previewStartDelay = data.config.cron.previewSchedule.runAtStartDelay this.actorRescrapeStartDelay = data.config.cron.actorRescrapeSchedule.runAtStartDelay this.stashdbRescrapeStartDelay = data.config.cron.stashdbRescrapeSchedule.runAtStartDelay - this.linkScenesStartDelay = data.config.cron.linkScenesSchedule.runAtStartDelay + this.linkScenesStartDelay = data.config.cron.linkScenesSchedule.runAtStartDelay + if (data.config.cron.organizeSchedule) { + this.organizeEnabled = data.config.cron.organizeSchedule.enabled + this.organizeHourInterval = data.config.cron.organizeSchedule.hourInterval + this.organizeStartDelay = data.config.cron.organizeSchedule.runAtStartDelay + } this.isLoading = false }) }, @@ -516,7 +543,14 @@ export default { linkScenesMinuteStart: this.linkScenesMinuteStart, linkScenesHourStart: this.linkScenesTimeRange[0], linkScenesHourEnd: this.linkScenesTimeRange[1], - linkScenesStartDelay:this.linkScenesStartDelay + linkScenesStartDelay:this.linkScenesStartDelay, + organizeEnabled: this.organizeEnabled, + organizeHourInterval: this.organizeHourInterval, + organizeUseRange: false, + organizeMinuteStart: 0, + organizeHourStart: 0, + organizeHourEnd: 23, + organizeStartDelay: this.organizeStartDelay } }) .json()