Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ OScraper is designed for people who want a controlled, auditable alternative to
- **Safe writes** — verify fresh directory fingerprints, reject stale previews, and never overwrite existing media paths.
- **Recoverable jobs** — run scraping through persistent bounded workers with per-operation checkpoints and retry support.
- **Media-server metadata** — generate movie, TV show, and episode NFO XML compatible with Kodi, Jellyfin, and Emby.
- **Multi-version movies** — organize same-directory or loose title/year variants into Jellyfin/Emby-style `Movie - Version.ext` files with editable version labels and shared `movie.nfo` metadata.
- **Operational visibility** — search job history, API logs, application logs, and administrator audit logs; export logs as CSV.
- **User-friendly interface** — responsive light/dark UI with English and Simplified Chinese.
- **Simple self-hosting** — one multi-platform image for `linux/amd64` and `linux/arm64`, backed by SQLite.
Expand Down
11 changes: 6 additions & 5 deletions backend/internal/handler/dto.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,14 +98,15 @@ func (r previewSearchRequest) command() service.SearchPreviewCommand {
}

type createPreviewRequest struct {
CandidateID uint `json:"candidate_id" binding:"required"`
TMDBID int `json:"tmdb_id" binding:"omitempty,min=1"`
Title string `json:"title" binding:"max=500"`
Year int `json:"year" binding:"omitempty,min=1870,max=2200"`
CandidateID uint `json:"candidate_id" binding:"required"`
TMDBID int `json:"tmdb_id" binding:"omitempty,min=1"`
Title string `json:"title" binding:"max=500"`
Year int `json:"year" binding:"omitempty,min=1870,max=2200"`
MovieVersionLabels map[string]string `json:"movie_version_labels" binding:"omitempty,max=8"`
}

func (r createPreviewRequest) command() service.CreatePreviewCommand {
return service.CreatePreviewCommand{CandidateID: r.CandidateID, TMDBID: r.TMDBID, Title: r.Title, Year: r.Year}
return service.CreatePreviewCommand{CandidateID: r.CandidateID, TMDBID: r.TMDBID, Title: r.Title, Year: r.Year, MovieVersionLabels: r.MovieVersionLabels}
}

type submitJobRequest struct {
Expand Down
148 changes: 148 additions & 0 deletions backend/internal/media/movie_version.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
package media

import (
"path"
"regexp"
"strconv"
"strings"
)

var (
movieVersionResolutionPattern = regexp.MustCompile(`(?i)(?:^|[ ._\-\[(])(4320p|2160p|1080p|720p|576p|480p|8k|4k|uhd)(?:$|[ ._\-\])])`)
movieVersionMultipartPattern = regexp.MustCompile(`(?i)(?:^|[ ._\-\[(])(?:cd|disc|disk|part|pt)[ ._\-]*\d+(?:$|[ ._\-\])])`)
movieVersionTechnicalPattern = regexp.MustCompile(`(?i)^(?:4320p|2160p|1080p|720p|576p|480p|8k|4k|uhd|hdr10\+?|hdr|dolby[ ._-]*vision|dv|bluray|blu-ray|web[ ._-]*dl|webrip|hdtv|remux|x26[45]|h26[45]|hevc|avc|aac|dts|flac|10bit)$`)
movieVersionDVPattern = regexp.MustCompile(`(?i)(?:^|[ ._\-])dv(?:$|[ ._\-])`)
movieVersionHDRPattern = regexp.MustCompile(`(?i)(?:^|[ ._\-])hdr(?:$|[ ._\-])`)
)

type MovieVersionLabel struct {
Label string
Source string
}

// MovieIdentityKey returns a conservative identity used to group loose movie
// files. A provider id wins; otherwise both a parsed title and year are
// required so unrelated root-level files are never grouped by title alone.
func MovieIdentityKey(info Info, uniqueFallback string) string {
if info.TMDBID != nil && *info.TMDBID > 0 {
return "tmdb:" + strconv.Itoa(*info.TMDBID)
}
if strings.TrimSpace(info.Title) != "" && info.Year != nil {
return "title:" + strings.ToLower(strings.Join(strings.Fields(info.Title), " ")) + ":" + strconv.Itoa(*info.Year)
}
return "file:" + strings.ToLower(uniqueFallback)
}

func IsMovieExtra(candidateRoot string, entryPath string) bool {
relative := strings.TrimPrefix(entryPath, strings.TrimRight(candidateRoot, "/")+"/")
for _, component := range strings.Split(strings.ToLower(path.Dir(relative)), "/") {
normalized := strings.NewReplacer("_", " ", "-", " ").Replace(component)
switch strings.Join(strings.Fields(normalized), " ") {
case "extras", "extra", "trailers", "trailer", "featurettes", "featurette", "samples", "sample", "behind the scenes", "deleted scenes", "interviews", "shorts":
return true
}
}
base := strings.ToLower(strings.TrimSuffix(path.Base(entryPath), path.Ext(entryPath)))
for _, suffix := range []string{"-trailer", ".trailer", "_trailer", "-sample", ".sample", "_sample", "-featurette", ".featurette", "_featurette"} {
if strings.HasSuffix(base, suffix) {
return true
}
}
return false
}

func IsMultipartMovieFile(name string) bool {
return movieVersionMultipartPattern.MatchString(strings.TrimSuffix(path.Base(name), path.Ext(name)))
}

// InferMovieVersionLabel creates stable, user-facing labels from common
// edition, resolution, HDR, source and remux tokens. Unknown text after an
// explicit " - " separator is retained as an explicit edition label.
func InferMovieVersionLabel(name string) MovieVersionLabel {
base := strings.TrimSpace(strings.TrimSuffix(path.Base(name), path.Ext(name)))
lower := strings.ToLower(base)
parts := make([]string, 0, 5)
appendPart := func(value string) {
for _, existing := range parts {
if strings.EqualFold(existing, value) {
return
}
}
parts = append(parts, value)
}

editions := []struct {
patterns []string
label string
}{
{[]string{"director's cut", "directors cut", "director cut"}, "Director's Cut"},
{[]string{"extended cut", "extended edition", "extended"}, "Extended"},
{[]string{"theatrical cut", "theatrical edition", "theatrical"}, "Theatrical"},
{[]string{"unrated cut", "unrated"}, "Unrated"},
{[]string{"open matte", "open.matte", "open_matte"}, "Open Matte"},
{[]string{"criterion"}, "Criterion"},
{[]string{"imax"}, "IMAX"},
}
for _, edition := range editions {
for _, marker := range edition.patterns {
if strings.Contains(lower, marker) {
appendPart(edition.label)
break
}
}
}
if match := movieVersionResolutionPattern.FindStringSubmatch(base); len(match) == 2 {
resolution := strings.ToLower(match[1])
switch resolution {
case "4k", "uhd":
resolution = "2160p"
case "8k":
resolution = "4320p"
}
appendPart(resolution)
}
if strings.Contains(lower, "dolby vision") || movieVersionDVPattern.MatchString(base) {
appendPart("Dolby Vision")
} else if strings.Contains(lower, "hdr10+") {
appendPart("HDR10+")
} else if strings.Contains(lower, "hdr10") {
appendPart("HDR10")
} else if movieVersionHDRPattern.MatchString(base) {
appendPart("HDR")
}
if strings.Contains(lower, "remux") {
appendPart("Remux")
} else if strings.Contains(lower, "blu-ray") || strings.Contains(lower, "bluray") {
appendPart("BluRay")
} else if strings.Contains(lower, "web-dl") || strings.Contains(lower, "web.dl") || strings.Contains(lower, "web_dl") {
appendPart("WEB-DL")
} else if strings.Contains(lower, "webrip") || strings.Contains(lower, "web-rip") {
appendPart("WEBRip")
} else if strings.Contains(lower, "hdtv") {
appendPart("HDTV")
}
if len(parts) > 0 {
source := "derived"
if strings.Contains(base, " - ") {
source = "explicit"
} else if len(parts) == 1 && movieVersionResolutionPattern.MatchString(base) {
source = "resolution"
}
return MovieVersionLabel{Label: strings.Join(parts, " "), Source: source}
}

if index := strings.LastIndex(base, " - "); index >= 0 {
suffix := strings.TrimSpace(base[index+3:])
words := strings.Fields(strings.NewReplacer(".", " ", "_", " ").Replace(suffix))
kept := make([]string, 0, len(words))
for _, word := range words {
if !movieVersionTechnicalPattern.MatchString(word) {
kept = append(kept, word)
}
}
if len(kept) > 0 {
return MovieVersionLabel{Label: strings.Join(kept, " "), Source: "explicit"}
}
}
return MovieVersionLabel{}
}
29 changes: 29 additions & 0 deletions backend/internal/media/parser_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,35 @@ func TestSeasonAliasesAndTMDBID(t *testing.T) {
}
}

func TestMovieVersionLabelInference(t *testing.T) {
tests := map[string]string{
"Movie (2020) - 2160p.x265.AAC.mkv": "2160p",
"Movie.2020.1080p.WEB-DL.mkv": "1080p WEB-DL",
"Movie (2020) - Director's Cut.2160p.HDR.Remux.mkv": "Director's Cut 2160p HDR Remux",
"Movie (2020) - Open.Matte.mkv": "Open Matte",
}
for name, expected := range tests {
if result := InferMovieVersionLabel(name); result.Label != expected {
t.Fatalf("%q: got %#v, want %q", name, result, expected)
}
}
if result := InferMovieVersionLabel("Movie (2020).mkv"); result.Label != "" {
t.Fatalf("plain movie unexpectedly received a version: %#v", result)
}
}

func TestMovieExtraAndMultipartClassification(t *testing.T) {
if !IsMovieExtra("/movies/Film", "/movies/Film/trailers/Film-trailer.mkv") {
t.Fatal("trailer directory was not classified as an extra")
}
if IsMovieExtra("/movies/Film", "/movies/Film/Film - 2160p.mkv") {
t.Fatal("main version was classified as an extra")
}
if !IsMultipartMovieFile("Film.CD1.mkv") || IsMultipartMovieFile("Film.2160p.mkv") {
t.Fatal("multipart classification failed")
}
}

func TestVideoExtensionsAreCaseInsensitive(t *testing.T) {
if !IsVideoFile("Film.MKV") || IsVideoFile("poster.jpg") {
t.Fatal("unexpected video extension classification")
Expand Down
113 changes: 105 additions & 8 deletions backend/internal/service/catalog.go
Original file line number Diff line number Diff line change
Expand Up @@ -386,7 +386,11 @@ func (s *CatalogService) InspectCandidate(ctx context.Context, targetID, candida
if !found {
return nil, Conflict("candidate.source_missing", "Media candidate no longer exists")
}
entries = relatedFlatAssets(candidate.Path, siblings)
if candidate.VideoCount > 1 {
entries = groupedFlatMovieAssets(candidate, siblings)
} else {
entries = relatedFlatAssets(candidate.Path, siblings)
}
} else {
state := &scanState{}
files, walkErr := s.walk(ctx, source, candidate.Path, candidate.Path, 1, refresh, state)
Expand All @@ -409,17 +413,46 @@ func (s *CatalogService) discover(ctx context.Context, target *model.ScrapeTarge
return nil, Conflict("scan.too_large", "Directory scan exceeds the safe entry limit")
}
candidates := make([]model.MediaCandidate, 0)
if target.LibraryType == "movie" {
groups := make(map[string]*flatMovieGroup)
for _, entry := range rootEntries {
if entry.IsDir || !media.IsVideoFile(entry.Name) || media.IsMovieExtra(target.RootPath, entry.Path) {
continue
}
info := media.ParseCandidate(strings.TrimSuffix(entry.Name, path.Ext(entry.Name)), entry.Name, target.LibraryType)
info = s.recognizeIfNeeded(ctx, info, entry.Name, entry.Name, target.LibraryType)
key := media.MovieIdentityKey(info, entry.Path)
group := groups[key]
if group == nil {
group = &flatMovieGroup{info: info}
groups[key] = group
}
group.videos = append(group.videos, entry)
}
keys := make([]string, 0, len(groups))
for key := range groups {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
group := groups[key]
sort.Slice(group.videos, func(left, right int) bool { return group.videos[left].Path < group.videos[right].Path })
assets := flatMovieGroupAssets(group.videos, rootEntries)
scraped := true
for _, video := range group.videos {
if !hasFlatScrapeMarker(video.Path, assets) {
scraped = false
break
}
}
representative := group.videos[0]
candidates = append(candidates, makeCandidate(representative.Path, target.LibraryType, group.info, representative.Name, len(group.videos), assets, scraped))
}
}
for _, entry := range rootEntries {
if err := ctx.Err(); err != nil {
return nil, Internal("scan.canceled", "Directory scan was canceled", err)
}
if target.LibraryType == "movie" && !entry.IsDir && media.IsVideoFile(entry.Name) {
info := media.ParseCandidate(strings.TrimSuffix(entry.Name, path.Ext(entry.Name)), entry.Name, target.LibraryType)
info = s.recognizeIfNeeded(ctx, info, entry.Name, entry.Name, target.LibraryType)
assets := relatedFlatAssets(entry.Path, rootEntries)
candidates = append(candidates, makeCandidate(entry.Path, target.LibraryType, info, entry.Name, 1, assets, hasFlatScrapeMarker(entry.Path, assets)))
continue
}
if !entry.IsDir {
continue
}
Expand All @@ -438,6 +471,70 @@ func (s *CatalogService) discover(ctx context.Context, target *model.ScrapeTarge
return candidates, nil
}

type flatMovieGroup struct {
info media.Info
videos []openlist.DirectoryEntry
}

func flatMovieGroupAssets(videos, siblings []openlist.DirectoryEntry) []openlist.DirectoryEntry {
videoPaths := make(map[string]struct{}, len(videos))
for _, video := range videos {
videoPaths[video.Path] = struct{}{}
}
assets := make([]openlist.DirectoryEntry, 0)
seen := make(map[string]struct{})
for _, video := range videos {
for _, entry := range relatedFlatAssets(video.Path, siblings) {
if media.IsVideoFile(entry.Name) {
if _, belongs := videoPaths[entry.Path]; !belongs {
continue
}
}
if _, exists := seen[entry.Path]; exists {
continue
}
seen[entry.Path] = struct{}{}
assets = append(assets, entry)
}
}
sort.Slice(assets, func(left, right int) bool { return assets[left].Path < assets[right].Path })
return assets
}

func groupedFlatMovieAssets(candidate *model.MediaCandidate, siblings []openlist.DirectoryEntry) []openlist.DirectoryEntry {
videos := make([]openlist.DirectoryEntry, 0)
seen := make(map[string]struct{})
wantedKey := media.MovieIdentityKey(media.Info{Title: candidate.ParsedTitle, Year: candidate.Year, TMDBID: candidate.TMDBID}, candidate.Path)
for _, entry := range siblings {
if entry.IsDir || !media.IsVideoFile(entry.Name) || media.IsMovieExtra(path.Dir(candidate.Path), entry.Path) {
continue
}
info := media.ParseCandidate(strings.TrimSuffix(entry.Name, path.Ext(entry.Name)), entry.Name, "movie")
if media.MovieIdentityKey(info, entry.Path) == wantedKey {
videos = append(videos, entry)
seen[entry.Path] = struct{}{}
}
}
var previous []openlist.DirectoryEntry
_ = json.Unmarshal([]byte(candidate.ManifestJSON), &previous)
previousVideos := make(map[string]struct{})
for _, entry := range previous {
if !entry.IsDir && media.IsVideoFile(entry.Name) {
previousVideos[entry.Path] = struct{}{}
}
}
for _, entry := range siblings {
if _, wasGrouped := previousVideos[entry.Path]; !wasGrouped {
continue
}
if _, exists := seen[entry.Path]; !exists {
videos = append(videos, entry)
seen[entry.Path] = struct{}{}
}
}
return flatMovieGroupAssets(videos, siblings)
}

func (s *CatalogService) recognizeIfNeeded(ctx context.Context, parsed media.Info, fileName, relativePath, libraryType string) media.Info {
if s.recognizer == nil || parsed.Confidence >= 70 || parsed.TMDBID != nil {
return parsed
Expand Down
Loading