From 140fb13908ae6ecc65aa327aa2f64e53b2f4aafe Mon Sep 17 00:00:00 2001 From: Ramon Ebdon Date: Thu, 5 Dec 2024 11:02:38 +1100 Subject: [PATCH 1/4] Resolve pg errors on orderby in count(*)-fix #645 --- src/repository/listRepository.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/repository/listRepository.ts b/src/repository/listRepository.ts index e2b3b7ae..5ce93864 100644 --- a/src/repository/listRepository.ts +++ b/src/repository/listRepository.ts @@ -326,6 +326,7 @@ export const listRepository = { .leftJoin('mediaItem', 'listItem.mediaItemId', 'mediaItem.id') .modify((qb) => filterQuery(qb, args)) .modify((qb) => sortQuery(qb, args)) + .clear('order') .count({ count: '*' }); const listItems: ListItemModel[] = await trx('listItem') From 183751c1f10b9109ff1c0c88bef217d3bdf79ba9 Mon Sep 17 00:00:00 2001 From: Ramon Ebdon Date: Fri, 6 Dec 2024 13:34:20 +1100 Subject: [PATCH 2/4] Import from CSV added to Imports page --- client/src/Router.tsx | 5 + client/src/pages/ImportPage.tsx | 1 + client/src/pages/import/ImportFromCsvPage.tsx | 73 +++++++++++++++ src/import/csvImport.ts | 91 +++++++++++++++++++ src/routers/importRouter.ts | 8 +- 5 files changed, 176 insertions(+), 2 deletions(-) create mode 100644 client/src/pages/import/ImportFromCsvPage.tsx create mode 100644 src/import/csvImport.ts diff --git a/client/src/Router.tsx b/client/src/Router.tsx index 285e10fd..245a0f7c 100644 --- a/client/src/Router.tsx +++ b/client/src/Router.tsx @@ -12,6 +12,7 @@ import { ImportFromFloxPage } from './pages/import/ImportFromFloxPage'; import { ImportFromGoodreadsPage } from './pages/import/ImportFromGoodreadsPage'; import { ImportFromSimklPage } from './pages/import/ImportFromSimklPage'; import { ImportFromTraktPage } from './pages/import/ImportFromTraktPage'; +import { ImportFromCsvPage } from './pages/import/ImportFromCsvPage'; import { ImportPage } from './pages/ImportPage'; import { JellyfinIntegrationPage } from './pages/integrations/JellyfinIntegrationPage'; import { KodiIntegrationPage } from './pages/integrations/KodiIntegrationPage'; @@ -146,6 +147,10 @@ export const router = createBrowserRouter([ path: '/import/mediatracker', element: , }, + { + path: '/import/csv', + element: , + }, { path: '/integrations', element: , diff --git a/client/src/pages/ImportPage.tsx b/client/src/pages/ImportPage.tsx index cb89bdce..4cb98bb3 100644 --- a/client/src/pages/ImportPage.tsx +++ b/client/src/pages/ImportPage.tsx @@ -25,6 +25,7 @@ export const ImportPage: FC = () => { + ); diff --git a/client/src/pages/import/ImportFromCsvPage.tsx b/client/src/pages/import/ImportFromCsvPage.tsx new file mode 100644 index 00000000..92aef12a --- /dev/null +++ b/client/src/pages/import/ImportFromCsvPage.tsx @@ -0,0 +1,73 @@ +import { FC } from 'react'; +import { ImportFormFilePage } from '../ImportFromFilePage'; +import { Trans } from '@lingui/macro'; +import { MainTitle } from '../../components/MainTitle'; + +export const ImportFromCsvPage: FC = () => { + return ( + <> + Import, from CSV]} + /> + + { + + }} + /> + + ); +}; + +const CsvInstructions: FC = () => { + + //workaround because react and the ligui lib process escapes differently + const newline = '\\n'; + + return ( +
+ + Click to show CSV file requirements + +
+
    +
  • Comma delimited, plain UTF-8 files only
    • +
  • Quote and escape characters are optional but must use double quotes: "
    • +
  • Windows (\r{newline}), Linux ({newline}) and old macOS (\r) record delimiters are auto-detected
    • +
  • The first line MUST be column headers
    • +
  • + Allowed column headers are, in any order and case-insensitive:
    + type, imdbId, tmdbId, tvdbId, listId, watched, season, episode +
    • +
  • The only mandatory column is type
    • +
  • + Other columns are optional, but you must include at least one of:
    + tmdbId, imdbId, tvdbId +
    • +
  • Leading and trailing whitespaces are stripped
    • +
  • Any record that cannot be parsed or contains errors will be skipped
    • +
  • List IDs must exist and be owned by the user
    • +
  • Items with invalid or other users' list IDs are discarded
    • +
  • The watchlist list ID is found on the Lists page
    • +
  • Watched is a Y/N column only
    • +
  • + Movies will be looked up in this order:
    + tmdbId, imdbId +
    • +
  • + TV shows will be looked up in this order:
    + tmdbId, imdbId, tvdbId +
    • +
  • TV shows must only use the show's main ID from tvdb, tmdb, or imdb
    • +
  • + To set episodes of a TV show as Watched, must provide a record for each:
    + season and episode +
    • +
+
+
+ ); +}; diff --git a/src/import/csvImport.ts b/src/import/csvImport.ts new file mode 100644 index 00000000..87d23814 --- /dev/null +++ b/src/import/csvImport.ts @@ -0,0 +1,91 @@ +import { parse } from 'csv-parse/sync'; +import _ from 'lodash'; +import { record, z } from 'zod'; + +import { ImportDataType, ImportListItem, ImportSeenHistoryItem, ImportWatchlistItem } from '../repository/importRepository.js'; +import { listRepository } from '../repository/listRepository.js'; +import { itemTypeSchema } from '../entity/mediaItemModel.js'; + +export const csvImport = { + async map(user: number, csvData: string): Promise { + const data = csvImportSchema.parse( + parse(csvData, { + bom: true, //detects and removes any utf-8 byte order marks + delimiter: ',', //did you know the "C" in CSV stands for "comma"? :) + columns: header => header.map((column: string) => column.toLowerCase()), + skip_empty_lines: true, //blank lines are ignored + skip_records_with_error: true, //if any field parsing errors (eg: invalid numbers), skip entire record + trim: true //strip leading and trailing whitespace in fields + }) + ); + + const dateNow = new Date(); + const userLists = await listRepository.getLists({userId: user}); + const watchListId = userLists.find(userList => userList.isWatchlist)?.id; + + const importlists = _(data) + .map((item) => item.listid) + .uniq() + .value(); + + const importData: ImportDataType = {}; + + importData.watchlist = data + .filter((item) => item.listid == watchListId) + .map((item) => ({ + itemType: item.type, + imdbId: item.imdbid, + tmdbId: item.tmdbid, + tvdbId: item.tvdbid, + addedAt: dateNow + })); + + importData.lists = userLists + .filter((list) => importlists.find(l => l == list.id && !list.isWatchlist)) + .map((list) => ({ + name: list.name, + description: list.description, + traktId: list.traktId, + createdAt: dateNow, + items: data + .filter((item) => item.listid == list.id) + .map((item) => ({ + itemType: item.type, + imdbId: item.imdbid, + tmdbId: item.tmdbid, + tvdbId: item.tvdbid, + addedAt: dateNow + })) + })); + + importData.seenHistory = data + .filter((item) => item.watched == 'Y') + .map((item) => ({ + itemType: item.type === 'tv' ? 'episode' : item.type, + imdbId: item.imdbid, + tmdbId: item.tmdbid, + tvdbId: item.tvdbid, + seenAt: dateNow, + episode: { + episodeNumber: item.episode, + seasonNumber: item.season + } + })); + + return importData; + }, +}; + +const csvImportSchema = z.array( + z.object({ + //lowercase every column to allow case insensitive parsing + type: z.enum(['tv', 'movie']), + imdbid: z.string().optional(), + tmdbid: z.coerce.number().optional(), + tvdbid: z.coerce.number().optional(), + listid: z.coerce.number().optional(), + watched: z.string().optional().default('N'), + season: z.coerce.number().optional(), + episode: z.coerce.number().optional() + }) +); diff --git a/src/routers/importRouter.ts b/src/routers/importRouter.ts index 93881077..968761cc 100644 --- a/src/routers/importRouter.ts +++ b/src/routers/importRouter.ts @@ -5,6 +5,7 @@ import { backupImport } from '../import/backupImport.js'; import { floxImport } from '../import/floxImport.js'; import { goodreadsImport } from '../import/goodreadsImport.js'; import { simklImport } from '../import/simklImport.js'; +import { csvImport } from '../import/csvImport.js'; import { importRepository, ImportState, @@ -17,6 +18,7 @@ const importSourceSchema = z.enum([ 'Goodreads', 'Simkl', 'MediaTracker', + 'CSV' ]); export type ImportSource = z.infer; @@ -80,7 +82,7 @@ const importFromFileHandler = async (args: { }); try { - const importData = await mapImportData(source, data); + const importData = await mapImportData(userId, source, data); await importRepository.importDataByExternalIds({ userId, @@ -123,7 +125,7 @@ const progressMap = (() => { return { set, get, remove, has }; })(); -const mapImportData = async (source: ImportSource, data: string) => { +const mapImportData = async (userId: number, source: ImportSource, data: string) => { switch (source) { case 'Flox': return floxImport.map(data); @@ -134,5 +136,7 @@ const mapImportData = async (source: ImportSource, data: string) => { return simklImport.map(data); case 'MediaTracker': return backupImport.map(data); + case 'CSV': + return csvImport.map(userId, data); } }; From 30e2ff347a97f51062333ff9e11e743422607283 Mon Sep 17 00:00:00 2001 From: Ramon Ebdon Date: Sat, 7 Dec 2024 23:58:32 +1100 Subject: [PATCH 3/4] csv import tidy up, add ratings, show errors --- client/src/pages/ImportFromFilePage.tsx | 3 +- client/src/pages/import/ImportFromCsvPage.tsx | 18 ++- src/import/csvImport.ts | 117 ++++++++++-------- 3 files changed, 84 insertions(+), 54 deletions(-) diff --git a/client/src/pages/ImportFromFilePage.tsx b/client/src/pages/ImportFromFilePage.tsx index 00faddda..a19b02f2 100644 --- a/client/src/pages/ImportFromFilePage.tsx +++ b/client/src/pages/ImportFromFilePage.tsx @@ -61,7 +61,8 @@ export const ImportFormFilePage: FC<{ /> {importFromFile.isError && (
- Unexpected file format + Unexpected file format
+ 
{ importFromFile.variables?.source == 'CSV' ? importFromFile.error.message : "" }
)} {file && ( diff --git a/client/src/pages/import/ImportFromCsvPage.tsx b/client/src/pages/import/ImportFromCsvPage.tsx index 92aef12a..7c1e4e2f 100644 --- a/client/src/pages/import/ImportFromCsvPage.tsx +++ b/client/src/pages/import/ImportFromCsvPage.tsx @@ -40,19 +40,24 @@ const CsvInstructions: FC = () => {
  • The first line MUST be column headers
  • Allowed column headers are, in any order and case-insensitive:
    - type, imdbId, tmdbId, tvdbId, listId, watched, season, episode + type, imdbId, tmdbId, tvdbId, listId, rating, seen, season, episode
  • The only mandatory column is type
    • +
  • + Valid values for type are:
    + tv, movie +
  • Other columns are optional, but you must include at least one of:
    tmdbId, imdbId, tvdbId
  • Leading and trailing whitespaces are stripped
  • Any record that cannot be parsed or contains errors will be skipped
    • +
  • Any record with missing fields compared to header will be skipped
  • List IDs must exist and be owned by the user
    • -
  • Items with invalid or other users' list IDs are discarded
    • +
  • Items with invalid or other users list IDs are discarded
  • The watchlist list ID is found on the Lists page
    • -
  • Watched is a Y/N column only
    • +
  • Seen is a Y/N column only
  • Movies will be looked up in this order:
    tmdbId, imdbId @@ -63,9 +68,14 @@ const CsvInstructions: FC = () => {
  • TV shows must only use the show's main ID from tvdb, tmdb, or imdb
  • - To set episodes of a TV show as Watched, must provide a record for each:
    + To set episodes of a TV show as Seen, must provide a record for each:
    season and episode
    • +
  • Valid ratings are decimal values between 0.1 and 10.0
    • +
  • + If you use "out of 5" ratings, multiply the value by 2
    + Eg: for a rating of 4 out of 5, provide a value of 8 +
    • diff --git a/src/import/csvImport.ts b/src/import/csvImport.ts index 87d23814..115a26a9 100644 --- a/src/import/csvImport.ts +++ b/src/import/csvImport.ts @@ -2,9 +2,9 @@ import { parse } from 'csv-parse/sync'; import _ from 'lodash'; import { record, z } from 'zod'; -import { ImportDataType, ImportListItem, ImportSeenHistoryItem, ImportWatchlistItem } from '../repository/importRepository.js'; +import { ImportDataType, ImportListItem, ImportRatingItem, ImportSeenHistoryItem, ImportWatchlistItem } from '../repository/importRepository.js'; import { listRepository } from '../repository/listRepository.js'; -import { itemTypeSchema } from '../entity/mediaItemModel.js'; +import { mediaTypeSchema } from '../entity/mediaItemModel.js'; export const csvImport = { async map(user: number, csvData: string): Promise { @@ -17,60 +17,78 @@ export const csvImport = { skip_records_with_error: true, //if any field parsing errors (eg: invalid numbers), skip entire record trim: true //strip leading and trailing whitespace in fields }) + ) + .filter(item => + (Object.values(mediaTypeSchema.Values).includes(item.type)) // sanity check + && ((item.type === 'tv' && (item.tmdbid || item.imdbid || item.tvdbid)) + ||(item.type === 'movie' && (item.tmdbid || item.imdbid))) ); - + const dateNow = new Date(); const userLists = await listRepository.getLists({userId: user}); const watchListId = userLists.find(userList => userList.isWatchlist)?.id; - const importlists = _(data) + const importLists = _(data) .map((item) => item.listid) .uniq() .value(); - const importData: ImportDataType = {}; - - importData.watchlist = data - .filter((item) => item.listid == watchListId) - .map((item) => ({ - itemType: item.type, - imdbId: item.imdbid, - tmdbId: item.tmdbid, - tvdbId: item.tvdbid, - addedAt: dateNow - })); - - importData.lists = userLists - .filter((list) => importlists.find(l => l == list.id && !list.isWatchlist)) - .map((list) => ({ - name: list.name, - description: list.description, - traktId: list.traktId, - createdAt: dateNow, - items: data - .filter((item) => item.listid == list.id) - .map((item) => ({ - itemType: item.type, - imdbId: item.imdbid, - tmdbId: item.tmdbid, - tvdbId: item.tvdbid, - addedAt: dateNow + const importData: ImportDataType = { + ratings: data + .filter((item) => item.rating && item.rating > 0) + .map((item) => ({ + itemType: item.type, + tmdbId: item.tmdbid ? item.tmdbid : undefined, + imdbId: item.imdbid ? item.imdbid : undefined, + tvdbId: item.tvdbid ? item.tvdbid : undefined, + rating: item.rating, + ratedAt: dateNow, + episode: item.type === 'tv' ? { + seasonNumber: item.season ? item.season : undefined, + episodeNumber: item.episode ? item.episode : undefined + } : undefined + })), + watchlist: data + .filter((item) => item.listid && watchListId && item.listid == watchListId) + .map((item) => ({ + itemType: item.type, + tmdbId: item.tmdbid ? item.tmdbid : undefined, + imdbId: item.imdbid ? item.imdbid : undefined, + tvdbId: item.tvdbid ? item.tvdbid : undefined, + addedAt: dateNow + })), + seenHistory: data + .filter((item) => item.seen == 'Y') + .map((item) => ({ + itemType: item.type === 'tv' ? 'episode' : item.type, + tmdbId: item.tmdbid ? item.tmdbid : undefined, + imdbId: item.imdbid ? item.imdbid : undefined, + tvdbId: item.tvdbid ? item.tvdbid : undefined, + seenAt: dateNow, + episode: item.type === 'tv' ? { + seasonNumber: item.season ? item.season : undefined, + episodeNumber: item.episode ? item.episode : undefined + } : undefined + })), + lists: userLists + .filter((userList) => importLists.find(importList => + !userList.isWatchlist && importList == userList.id)) + .map((list) => ({ + name: list.name, + description: list.description, + traktId: list.traktId, + createdAt: dateNow, + items: data + .filter((item) => item.listid == list.id) + .map((item) => ({ + itemType: item.type, + tmdbId: item.tmdbid ? item.tmdbid : undefined, + imdbId: item.imdbid ? item.imdbid : undefined, + tvdbId: item.tvdbid ? item.tvdbid : undefined, + addedAt: dateNow })) - })); - - importData.seenHistory = data - .filter((item) => item.watched == 'Y') - .map((item) => ({ - itemType: item.type === 'tv' ? 'episode' : item.type, - imdbId: item.imdbid, - tmdbId: item.tmdbid, - tvdbId: item.tvdbid, - seenAt: dateNow, - episode: { - episodeNumber: item.episode, - seasonNumber: item.season - } - })); + })) + } return importData; }, @@ -78,13 +96,14 @@ export const csvImport = { const csvImportSchema = z.array( z.object({ - //lowercase every column to allow case insensitive parsing + //lowercase every column name to allow case insensitive parsing type: z.enum(['tv', 'movie']), - imdbid: z.string().optional(), tmdbid: z.coerce.number().optional(), + imdbid: z.string().optional(), tvdbid: z.coerce.number().optional(), listid: z.coerce.number().optional(), - watched: z.string().optional().default('N'), + rating: z.coerce.number().optional(), + seen: z.string().optional(), season: z.coerce.number().optional(), episode: z.coerce.number().optional() }) From 400c5376f552df3f70407e23274a22ad255e8734 Mon Sep 17 00:00:00 2001 From: Ramon Ebdon Date: Wed, 8 Jan 2025 13:38:36 +1100 Subject: [PATCH 4/4] add search by igdbId to search page --- client/src/pages/SearchPage.tsx | 31 +++++++++++++++++++++++++------ src/routers/searchRouter.ts | 5 +++++ 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/client/src/pages/SearchPage.tsx b/client/src/pages/SearchPage.tsx index 31da8855..d92e69aa 100644 --- a/client/src/pages/SearchPage.tsx +++ b/client/src/pages/SearchPage.tsx @@ -55,6 +55,7 @@ const SearchPageImpl: FC<{ mediaType: MediaType }> = (props) => { narrator: ['audiobook'], tmdbId: ['movie', 'tv'], imdbId: ['movie', 'tv'], + igdbId: ['video_game'] }); const [query, setQuery] = useState({ @@ -63,6 +64,7 @@ const SearchPageImpl: FC<{ mediaType: MediaType }> = (props) => { narrator: searchParams.get('narrator'), imdbId: searchParams.get('imdbId'), tmdbId: parseInt(searchParams.get('tmdbId') || '') || null, + igdbId: parseInt(searchParams.get('igdbId') || '') || null, }); const searchQuery = trpc.search.search.useQuery( @@ -106,6 +108,7 @@ const SearchPageImpl: FC<{ mediaType: MediaType }> = (props) => { narrator: string; imdbId: string; tmdbId: string; + igdbId: string; }> className="flex flex-col gap-5 md:flex-row" validation={ @@ -120,9 +123,10 @@ const SearchPageImpl: FC<{ mediaType: MediaType }> = (props) => { }, } : undefined - } + } initialValues={{ tmdbId: query.tmdbId?.toString() || undefined, + igdbId: query.igdbId?.toString() || undefined, author: query.author || undefined, imdbId: query.imdbId || undefined, narrator: query.narrator || undefined, @@ -137,6 +141,9 @@ const SearchPageImpl: FC<{ mediaType: MediaType }> = (props) => { data.tmdbId?.length > 0 ? !isNaN(parseInt(data.tmdbId || '')) : false, + data.igdbId?.length > 0 + ? !isNaN(parseInt(data.igdbId || '')) + : false, data.imdbId, ].filter(Boolean).length === 0 ) { @@ -152,6 +159,7 @@ const SearchPageImpl: FC<{ mediaType: MediaType }> = (props) => { narrator: data.narrator || null, imdbId: data.imdbId || null, tmdbId: parseInt(data.tmdbId || '') || null, + igdbId: parseInt(data.igdbId || '') || null, }; setQuery(newQuery); @@ -171,7 +179,7 @@ const SearchPageImpl: FC<{ mediaType: MediaType }> = (props) => { = (props) => { narrator: null, query: null, tmdbId: null, + igdbId: null, }); }} /> @@ -190,7 +199,7 @@ const SearchPageImpl: FC<{ mediaType: MediaType }> = (props) => { {canUse('author') && ( = (props) => { {canUse('narrator') && ( = (props) => { {canUse('tmdbId') && ( = (props) => { {canUse('imdbId') && ( )} + {canUse('igdbId') && ( + + )} +