diff --git a/CHANGES.md b/CHANGES.md index 7d208cc04c23..82b19c40fcda 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -145,6 +145,7 @@ - SCAN: Interpret M3U files during scanning - SCAN: Merge manual and automatic scan into a more flexible setup - SCAN: Improve GDI content recognition +- SCAN: --scan runs every path and exits with what the scans did; Manual Scan options, --scan-refresh and --scan-refresh-all on the command line; Refresh All Playlists in Manage Playlists - SHADER: Display on-screen error when preset load fails - SHADER: Add gyroscope / accelerometer uniforms for shaders - SHADER: Add aggressive shader caching to slang shader backend diff --git a/core_info.c b/core_info.c index 1fb2138d5f8e..99c54d3b99c8 100644 --- a/core_info.c +++ b/core_info.c @@ -1648,7 +1648,7 @@ static bool core_info_path_is_standalone_exempt( return false; } -static size_t core_info_get_file_id(const char *core_filename, +size_t core_info_get_file_id(const char *core_filename, char *s, size_t len) { size_t _len; diff --git a/core_info.h b/core_info.h index fe3e2a3b7cb1..9740b6aca1de 100644 --- a/core_info.h +++ b/core_info.h @@ -200,6 +200,11 @@ bool core_info_list_update_missing_firmware(core_info_ctx_firmware_t *info); bool core_info_find(const char *core_path, core_info_t **core_info); +/* Writes the file id of 'core_filename' (no extension or + * platform suffix) to 's'; returns its length */ +size_t core_info_get_file_id(const char *core_filename, + char *s, size_t len); + bool core_info_load(const char *core_path); /* The extensions of every core that claims the database at diff --git a/docs/retroarch.6 b/docs/retroarch.6 index 342756d5deb9..1e44693c7f07 100644 --- a/docs/retroarch.6 +++ b/docs/retroarch.6 @@ -65,6 +65,68 @@ Prints available features compiled into RetroArch, then exits. \fB--cloudsync\fR Runs a cloud sync with the configured cloud sync driver, then exits. +.TP +\fB--scan PATH\fR +Scans the folder or file PATH for content and adds it to playlists, as Manual Scan does. +Can be given more than once. + +.TP +\fB--scan-refresh NAME|PATH\fR +Rescans one playlist with the scan settings it saved, as Refresh Playlist does. +NAME matches a playlist's name exactly first, then ignoring case. PATH must be in the playlist folder. +Like Refresh Playlist, it drops entries whose files are gone or whose extension the saved settings no longer include. +A playlist whose saved content folder is missing is skipped instead, so an unplugged drive does not empty it. + +.TP +\fB--scan-refresh-all\fR +Does --scan-refresh for every playlist that has saved scan settings, as Refresh All Playlists does. + +.IP +These run in the order given, and one that fails does not stop the rest. +Each prints a totals line on stdout, or its error on stderr. +Without --menu, RetroArch exits when they are done and does not load content given on the command line. +The exit status is then 1 if any failed, 0 if any added an entry, and 2 if none added anything; a skipped playlist does not count. +With --menu, they run while the menu does. +They cannot be combined with --cloudsync or --command. + +.IP +The options below set the Manual Scan settings of the --scan, --scan-refresh or --scan-refresh-all before them. +On a refresh they override the saved settings, and the refresh saves them. +A --scan without any is Fully Automatic. + +.TP +\fB--scan-playlist NAME\fR +Sets Target Playlist: a database's name, or a custom playlist name. Not allowed on a refresh. + +.TP +\fB--scan-database NAME|first\fR +Sets Database To Match: the database NAME, or the first database that matches. + +.TP +\fB--scan-match strict|loose|none\fR +Sets Database Check. + +.TP +\fB--scan-dat FILE\fR +Sets Arcade DAT File. Matching is strict unless --scan-match loose is given. + +.TP +\fB--scan-core CORE\fR +Sets Default Core: a core's path, its file name (with or without _libretro and the extension) or its display name. + +.TP +\fB--scan-exts EXTS\fR +Sets File Extensions, separated by commas or spaces. + +.TP +\fB--scan-no-recursive, --scan-no-archives\fR +Turn off Scan Recursively and Scan Inside Archives. + +.TP +\fB--scan-overwrite, --scan-validate, --scan-omit-db-ref\fR +Turn on Overwrite Existing Playlist, Validate Existing Entries and Skip Database References From Playlist. +They act only when the scan has one playlist, and --scan-overwrite only on a folder; otherwise they print a warning and are ignored. + .TP \fB-L PATH, --libretro PATH\fR Path to a libretro implementation which is to be used. diff --git a/intl/msg_hash_lbl.h b/intl/msg_hash_lbl.h index c6dc28b3a804..bd8f511833f3 100644 --- a/intl/msg_hash_lbl.h +++ b/intl/msg_hash_lbl.h @@ -1955,6 +1955,10 @@ MSG_HASH( MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_PLAYLIST, "playlist_manager_refresh_playlist" ) +MSG_HASH( + MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_ALL_PLAYLISTS, + "playlist_manager_refresh_all_playlists" + ) MSG_HASH( MENU_ENUM_LABEL_PLAYLIST_SETTINGS_BEGIN, "playlist_settings_begin" diff --git a/intl/msg_hash_us.h b/intl/msg_hash_us.h index 4a66492a66c5..57b61aadaf52 100644 --- a/intl/msg_hash_us.h +++ b/intl/msg_hash_us.h @@ -23216,6 +23216,14 @@ MSG_HASH( MENU_ENUM_SUBLABEL_PLAYLIST_MANAGER_REFRESH_PLAYLIST, "Add new content and remove invalid entries by repeating the content scan operation last used to create or edit the playlist." ) +MSG_HASH( + MENU_ENUM_LABEL_VALUE_PLAYLIST_MANAGER_REFRESH_ALL_PLAYLISTS, + "Refresh All Playlists" + ) +MSG_HASH( + MENU_ENUM_SUBLABEL_PLAYLIST_MANAGER_REFRESH_ALL_PLAYLISTS, + "Rescan every playlist that has saved scan settings, with those settings." + ) MSG_HASH( MENU_ENUM_LABEL_VALUE_DELETE_PLAYLIST, "Delete Playlist" @@ -32548,6 +32556,10 @@ MSG_HASH( MSG_PLAYLIST_MANAGER_REFRESH_DAT_FILE_TOO_LARGE, "Refresh failed - arcade DAT file too large (insufficient memory): " ) +MSG_HASH( + MSG_PLAYLIST_MANAGER_REFRESH_ALL_DONE, + "Playlists refreshed: %u, skipped: %u, failed: %u" + ) MSG_HASH( MSG_ADDED_TO_FAVORITES, "Added to favorites" @@ -34344,6 +34356,10 @@ MSG_HASH( MSG_MANUAL_CONTENT_SCAN_END, "Scan complete: " ) +MSG_HASH( + MSG_MANUAL_CONTENT_SCAN_ALREADY_RUNNING, + "A content scan is already running." + ) MSG_HASH( MSG_CORE_BACKUP_SCANNING_CORE, "Scanning core: " diff --git a/libretro-common/file/file_path.c b/libretro-common/file/file_path.c index 429baf04962b..9bc43fdbfa94 100644 --- a/libretro-common/file/file_path.c +++ b/libretro-common/file/file_path.c @@ -1120,6 +1120,7 @@ size_t fill_pathname_join_delim(char *s, const char *dir, size_t fill_pathname_expand_special(char *s, const char *in_path, size_t len) { #if !defined(RARCH_CONSOLE) && defined(RARCH_INTERNAL) + char *start = s; if (in_path[0] == '~' || in_path[0] == ':') { char app_dir[DIR_MAX_LENGTH]; @@ -1134,8 +1135,13 @@ size_t fill_pathname_expand_special(char *s, const char *in_path, size_t len) if (*app_dir) { + /* strlcpy() reports the source's length whether or not it + * fit: a directory longer than @s leaves the result cut + * there, with nothing appended past the end. */ size_t _len = strlcpy(s, app_dir, len); + if (_len >= len) + return len ? len - 1 : 0; s += _len; len -= _len; @@ -1143,15 +1149,21 @@ size_t fill_pathname_expand_special(char *s, const char *in_path, size_t len) { _len = strlcpy(s, PATH_DEFAULT_SLASH(), len); + if (_len >= len) + return (size_t)(s - start) + (len ? len - 1 : 0); s += _len; len -= _len; } - in_path += 2; + in_path++; + if (PATH_CHAR_IS_SLASH(*in_path)) + in_path++; } } -#endif + return (size_t)(s - start) + strlcpy(s, in_path, len); +#else return strlcpy(s, in_path, len); +#endif } size_t fill_pathname_abbreviate_special(char *s, diff --git a/libretro-common/samples/file/file_path/fill_pathname_test.c b/libretro-common/samples/file/file_path/fill_pathname_test.c index 4757476c8a37..deb4de036d0a 100644 --- a/libretro-common/samples/file/file_path/fill_pathname_test.c +++ b/libretro-common/samples/file/file_path/fill_pathname_test.c @@ -259,6 +259,16 @@ static void test_join_truncation(void) free(region); } +/* mingw-w64 has no setenv(). */ +static void set_home(const char *home) +{ +#ifdef _WIN32 + _putenv_s("HOME", home); +#else + setenv("HOME", home, 1); +#endif +} + /* fill_pathname_abbreviated_or_relative() resolves a relative @in_path * against @in_refpath's directory before choosing between the * relative and abbreviated forms. That join still carries its '..' @@ -273,7 +283,7 @@ static void test_abbreviated_or_relative_long_join(void) size_t _len = 0; size_t i; - setenv("HOME", "/nonexistent-home", 1); + set_home("/nonexistent-home"); _len = strlcpy(refpath, "/r/s/t/u/v/w/", sizeof(refpath)); for (i = 0; i < 230; i++) @@ -315,6 +325,80 @@ static void test_abbreviated_or_relative_long_join(void) printf("[SUCCESS] absolute reference preferred over deeper relative\n"); } +/* Equal but for which slash separates. */ +static int path_same(const char *a, const char *b) +{ + for (; *a && *b; a++, b++) + if (*a != *b && !(PATH_CHAR_IS_SLASH(*a) && PATH_CHAR_IS_SLASH(*b))) + return 0; + return *a == *b; +} + +/* fill_pathname_expand_special() replaces '~' and the slash after it. + * The input is a heap copy, so ASan sees a read past its NUL. */ +static void test_expand_special_home(void) +{ + static const char *const in[] = + { "~", "~/", "~/x", "~x", "", "a/~" }; + static const char *const want[] = + { "/h/", "/h/", "/h/x", "/h/x", "", "a/~" }; + char out[64]; + size_t i; + + set_home("/h"); + for (i = 0; i < sizeof(in) / sizeof(in[0]); i++) + { + size_t n = strlen(in[i]) + 1; + char *path = (char*)malloc(n); + if (!path) + { + printf("[FAILED] expand \"%s\": out of memory\n", in[i]); + failures++; + continue; + } + memcpy(path, in[i], n); + fill_pathname_expand_special(out, path, sizeof(out)); + free(path); + if (!path_same(out, want[i])) + { + printf("[FAILED] expand \"%s\" gave \"%s\" (expected \"%s\")\n", + in[i], out, want[i]); + failures++; + } + else + printf("[SUCCESS] expand \"%s\"\n", in[i]); + } +} + +/* A home directory longer than the buffer: the expansion is cut at the + * buffer's end and nothing is written past it. The buffer is a heap + * block of exactly its size, so ASan sees a write past the end. */ +static void test_expand_special_truncation(void) +{ + const size_t len = 8; + char *out = (char*)malloc(len); + size_t got; + + if (!out) + { + printf("[FAILED] expand truncation: out of memory\n"); + failures++; + return; + } + set_home("/a-home-directory-longer-than-the-buffer"); + got = fill_pathname_expand_special(out, "~/file", len); + if (got != len - 1 || strlen(out) != len - 1 + || strncmp(out, "/a-home", len - 1)) + { + printf("[FAILED] expand truncation gave \"%s\" (%u)\n", + out, (unsigned)got); + failures++; + } + else + printf("[SUCCESS] expand truncation\n"); + free(out); +} + int main(void) { /* Documented semantics. */ @@ -335,6 +419,8 @@ int main(void) test_overlong_input_no_dot(); test_join_truncation(); test_abbreviated_or_relative_long_join(); + test_expand_special_home(); + test_expand_special_truncation(); if (failures) { diff --git a/manual_content_scan.c b/manual_content_scan.c index d5368dbc4212..cbcc1c1d63c0 100644 --- a/manual_content_scan.c +++ b/manual_content_scan.c @@ -58,6 +58,20 @@ typedef struct bool validate_entries; } scan_settings_t; +#define MANUAL_CONTENT_SCAN_SETTINGS_DEFAULT { \ + MANUAL_CONTENT_SCAN_METHOD_AUTOMATIC, /* scan method */ \ + MANUAL_CONTENT_SCAN_SYSTEM_NAME_AUTO, /* system_name_type */ \ + MANUAL_CONTENT_SCAN_CORE_DETECT, /* core_type */ \ + MANUAL_CONTENT_SCAN_USE_DB_STRICT, \ + MANUAL_CONTENT_SCAN_SELECT_DB_AUTO, \ + true, /* search_recursively */ \ + true, /* search_archives */ \ + false, \ + false, /* filter_dat_content */ \ + false, /* overwrite_playlist */ \ + false /* validate_entries */ \ +} + /* TODO/FIXME - static public global variables */ /* Static settings object * > Provides easy access to settings parameters @@ -68,19 +82,7 @@ typedef struct * but this should not be an issue (i.e. regular configuration_settings * are not thread safe, but we only access them when pushing a * task, not in the task thread itself, so all is well) */ -static scan_settings_t scan_settings = { - MANUAL_CONTENT_SCAN_METHOD_AUTOMATIC, /* scan method */ - MANUAL_CONTENT_SCAN_SYSTEM_NAME_AUTO, /* system_name_type */ - MANUAL_CONTENT_SCAN_CORE_DETECT, /* core_type */ - MANUAL_CONTENT_SCAN_USE_DB_STRICT, - MANUAL_CONTENT_SCAN_SELECT_DB_AUTO, - true, /* search_recursively */ - true, /* search_archives */ - false, - false, /* filter_dat_content */ - false, /* overwrite_playlist */ - false /* validate_entries */ -}; +static scan_settings_t scan_settings = MANUAL_CONTENT_SCAN_SETTINGS_DEFAULT; /* The path and name buffers live outside the struct: three * non-zero defaults were dragging eleven kilobytes of zero buffers @@ -268,24 +270,30 @@ void manual_content_scan_scrub_file_exts_custom(void) * in, structs out - so both the extension whitelist * and the file system checks live here, with the scan * configuration that owns the path. */ -bool manual_content_scan_dat_path_is_valid( - const char *path, uint64_t *file_size) +/* The extensions a Logiqx XML DAT may legitimately carry: + * .dat and .xml */ +static bool manual_content_scan_dat_ext_is_valid(const char *path) { const char *file_ext = NULL; - int64_t file_size_int; if (!path || !*path) return false; - /* Check file extension (.dat / .xml, the extensions - * a Logiqx XML DAT may legitimately carry) */ file_ext = path_get_extension(path); if (!file_ext || !*file_ext) return false; - if ( !string_is_equal_noncase(file_ext, "dat") - && !string_is_equal_noncase(file_ext, "xml")) + return string_is_equal_noncase(file_ext, "dat") + || string_is_equal_noncase(file_ext, "xml"); +} + +bool manual_content_scan_dat_path_is_valid( + const char *path, uint64_t *file_size) +{ + int64_t file_size_int; + + if (!manual_content_scan_dat_ext_is_valid(path)) return false; /* Ensure file exists */ @@ -305,9 +313,11 @@ bool manual_content_scan_dat_path_is_valid( } /* Checks 'dat_file_path' string and resets it - * if invalid */ -enum manual_content_scan_dat_file_path_status - manual_content_scan_validate_dat_file_path(void) + * if invalid. @known_size, when not NULL, is the file's + * size as read already (<= 0: not there), so the file + * system is not asked again. */ +static enum manual_content_scan_dat_file_path_status + manual_content_scan_validate_dat(const int64_t *known_size) { enum manual_content_scan_dat_file_path_status dat_file_path_status = MANUAL_CONTENT_SCAN_DAT_FILE_UNSET; @@ -315,10 +325,22 @@ enum manual_content_scan_dat_file_path_status /* Check if 'dat_file_path' has been set */ if (*scan_dat_file_path) { - uint64_t file_size; + uint64_t file_size = 0; + bool valid; /* Check if path itself is valid */ - if (manual_content_scan_dat_path_is_valid(scan_dat_file_path, &file_size)) + if (known_size) + { + valid = manual_content_scan_dat_ext_is_valid(scan_dat_file_path) + && *known_size > 0; + if (valid) + file_size = (uint64_t)*known_size; + } + else + valid = manual_content_scan_dat_path_is_valid(scan_dat_file_path, + &file_size); + + if (valid) { uint64_t free_memory = mem_stats_free(); dat_file_path_status = MANUAL_CONTENT_SCAN_DAT_FILE_OK; @@ -359,12 +381,91 @@ enum manual_content_scan_dat_file_path_status return dat_file_path_status; } +enum manual_content_scan_dat_file_path_status + manual_content_scan_validate_dat_file_path(void) +{ + return manual_content_scan_validate_dat(NULL); +} + /* Menu setters */ +/* Sets every setting back to its default */ +void manual_content_scan_reset_settings(void) +{ + static const scan_settings_t d = MANUAL_CONTENT_SCAN_SETTINGS_DEFAULT; + + scan_settings = d; + scan_core_path[0] = '\0'; + scan_file_exts_core[0] = '\0'; + scan_file_exts_custom[0] = '\0'; + scan_dat_file_path[0] = '\0'; + scan_content_dir[0] = '\0'; + scan_system_name_content_dir[0] = '\0'; + scan_system_name_database[0] = '\0'; + scan_system_name_custom[0] = '\0'; + scan_core_name[0] = '\0'; +} + +struct manual_content_scan_saved +{ + scan_settings_t settings; + char core_path[PATH_MAX_LENGTH]; + char file_exts_core[PATH_MAX_LENGTH]; + char file_exts_custom[PATH_MAX_LENGTH]; + char dat_file_path[PATH_MAX_LENGTH]; + char content_dir[PATH_MAX_LENGTH]; + char system_name_content_dir[DIR_MAX_LENGTH]; + char system_name_database[NAME_MAX_LENGTH]; + char system_name_custom[NAME_MAX_LENGTH]; + char core_name[NAME_MAX_LENGTH]; +}; + +manual_content_scan_saved_t *manual_content_scan_save_settings(void) +{ + manual_content_scan_saved_t *sv = (manual_content_scan_saved_t*) + malloc(sizeof(*sv)); + if (!sv) + return NULL; + sv->settings = scan_settings; + memcpy(sv->core_path, scan_core_path, sizeof(sv->core_path)); + memcpy(sv->file_exts_core, scan_file_exts_core, sizeof(sv->file_exts_core)); + memcpy(sv->file_exts_custom, scan_file_exts_custom, sizeof(sv->file_exts_custom)); + memcpy(sv->dat_file_path, scan_dat_file_path, sizeof(sv->dat_file_path)); + memcpy(sv->content_dir, scan_content_dir, sizeof(sv->content_dir)); + memcpy(sv->system_name_content_dir, scan_system_name_content_dir, sizeof(sv->system_name_content_dir)); + memcpy(sv->system_name_database, scan_system_name_database, sizeof(sv->system_name_database)); + memcpy(sv->system_name_custom, scan_system_name_custom, sizeof(sv->system_name_custom)); + memcpy(sv->core_name, scan_core_name, sizeof(sv->core_name)); + return sv; +} + +void manual_content_scan_restore_settings(manual_content_scan_saved_t *sv) +{ + if (!sv) + return; + scan_settings = sv->settings; + memcpy(scan_core_path, sv->core_path, sizeof(scan_core_path)); + memcpy(scan_file_exts_core, sv->file_exts_core, sizeof(scan_file_exts_core)); + memcpy(scan_file_exts_custom, sv->file_exts_custom, sizeof(scan_file_exts_custom)); + memcpy(scan_dat_file_path, sv->dat_file_path, sizeof(scan_dat_file_path)); + memcpy(scan_content_dir, sv->content_dir, sizeof(scan_content_dir)); + memcpy(scan_system_name_content_dir, sv->system_name_content_dir, sizeof(scan_system_name_content_dir)); + memcpy(scan_system_name_database, sv->system_name_database, sizeof(scan_system_name_database)); + memcpy(scan_system_name_custom, sv->system_name_custom, sizeof(scan_system_name_custom)); + memcpy(scan_core_name, sv->core_name, sizeof(scan_core_name)); + free(sv); +} + /* Sets content directory for next manual scan * operation. * Returns true if content directory is valid. */ bool manual_content_scan_set_menu_content_dir(const char *content_dir) +{ + return manual_content_scan_set_menu_content_dir_fs(content_dir, NULL); +} + +bool manual_content_scan_set_menu_content_dir_fs(const char *content_dir, + const manual_content_scan_fs_t *fs) { size_t _len; int path_stats; @@ -426,7 +527,7 @@ bool manual_content_scan_set_menu_content_dir(const char *content_dir) /* A file takes the name of the folder that holds it, * as a scan of that folder would */ - path_stats = path_stat(scan_content_dir); + path_stats = fs ? fs->content_stat : path_stat(scan_content_dir); if ( (path_stats & RETRO_VFS_STAT_IS_VALID) && !(path_stats & RETRO_VFS_STAT_IS_DIRECTORY) && fill_pathname_parent_dir_name(scan_system_name_content_dir, @@ -555,6 +656,53 @@ bool manual_content_scan_set_menu_system_name( return false; } +/* Copies the core's path, name and extensions + * to the settings. + * Returns false if the core has no path. */ +static bool manual_content_scan_set_core(const core_info_t *core_info) +{ + /* Copy core path to settings struct */ + if (!core_info->path || !*core_info->path) + return false; + + strlcpy( + scan_core_path, + core_info->path, + sizeof(scan_core_path)); + + /* Copy core name to settings struct */ + strlcpy( + scan_core_name, + core_info->display_name, + sizeof(scan_core_name)); + + /* Copy supported extensions to settings + * struct, if required */ + if ( core_info->supported_extensions + && *core_info->supported_extensions) + { + strlcpy( + scan_file_exts_core, + core_info->supported_extensions, + sizeof(scan_file_exts_core)); + + /* Core info extensions are delimited by + * vertical bars. For internal consistency, + * replace them with spaces */ + string_replace_all_chars(scan_file_exts_core, '|', ' '); + + /* Apply standard scrubbing/clean-up + * (should not be required, but must handle the + * case where a core info file is incorrectly + * formatted) */ + manual_content_scan_scrub_file_exts(scan_file_exts_core); + } + else + scan_file_exts_core[0] = '\0'; + + return true; +} + /* Sets core name for the next manual scan * operation (+ core path and other associated * parameters). @@ -607,45 +755,9 @@ bool manual_content_scan_set_menu_core_name( /* Core has been found */ core_found = true; - /* Copy core path to settings struct */ - if (!core_info->path || !*core_info->path) + if (!manual_content_scan_set_core(core_info)) goto error; - strlcpy( - scan_core_path, - core_info->path, - sizeof(scan_core_path)); - - /* Copy core name to settings struct */ - strlcpy( - scan_core_name, - core_info->display_name, - sizeof(scan_core_name)); - - /* Copy supported extensions to settings - * struct, if required */ - if ( core_info->supported_extensions - && *core_info->supported_extensions) - { - strlcpy( - scan_file_exts_core, - core_info->supported_extensions, - sizeof(scan_file_exts_core)); - - /* Core info extensions are delimited by - * vertical bars. For internal consistency, - * replace them with spaces */ - string_replace_all_chars(scan_file_exts_core, '|', ' '); - - /* Apply standard scrubbing/clean-up - * (should not be required, but must handle the - * case where a core info file is incorrectly - * formatted) */ - manual_content_scan_scrub_file_exts(scan_file_exts_core); - } - else - scan_file_exts_core[0] = '\0'; - break; } } @@ -674,8 +786,17 @@ bool manual_content_scan_set_menu_core_name( * Returns MANUAL_CONTENT_SCAN_PLAYLIST_REFRESH_OK * if playlist contains a valid scan record. */ enum manual_content_scan_playlist_refresh_status - manual_content_scan_set_menu_from_playlist(playlist_t *playlist, - const char *path_content_database, bool show_hidden_files) + manual_content_scan_set_menu_from_playlist_list(playlist_t *playlist, + const struct string_list *rdb_list) +{ + return manual_content_scan_set_menu_from_playlist_list_fs(playlist, + rdb_list, NULL); +} + +enum manual_content_scan_playlist_refresh_status + manual_content_scan_set_menu_from_playlist_list_fs( + playlist_t *playlist, const struct string_list *rdb_list, + const manual_content_scan_fs_t *fs) { const char *playlist_path = NULL; const char *content_dir = NULL; @@ -689,9 +810,6 @@ enum manual_content_scan_playlist_refresh_status bool overwrite_playlist = false; bool omit_db_ref = false; int db_usage = MANUAL_CONTENT_SCAN_USE_DB_NONE; -#ifdef HAVE_LIBRETRODB - struct string_list *rdb_list = NULL; -#endif enum manual_content_scan_system_name_type system_name_type = MANUAL_CONTENT_SCAN_SYSTEM_NAME_CONTENT_DIR; enum manual_content_scan_core_type @@ -729,13 +847,15 @@ enum manual_content_scan_playlist_refresh_status if (database_name_lpl && *database_name_lpl) fill_pathname(database_name,database_name_lpl,"",sizeof(database_name)); + else + database_name[0] = '\0'; /* Cannot happen, but would constitute a * 'system name' error */ if (!*system_name) return MANUAL_CONTENT_SCAN_PLAYLIST_REFRESH_INVALID_SYSTEM_NAME; /* Set content directory */ - if (!manual_content_scan_set_menu_content_dir(content_dir)) + if (!manual_content_scan_set_menu_content_dir_fs(content_dir, fs)) return MANUAL_CONTENT_SCAN_PLAYLIST_REFRESH_INVALID_CONTENT_DIR; if (!manual_content_scan_set_menu_scan_use_db((enum manual_content_scan_db_usage)db_usage)) @@ -744,13 +864,6 @@ enum manual_content_scan_playlist_refresh_status scan_settings.db_selection = MANUAL_CONTENT_SCAN_SELECT_DB_AUTO; /* Set system name */ -#ifdef HAVE_LIBRETRODB - /* > If platform has database support, get names - * of all installed database files */ - rdb_list = dir_list_new_special( - path_content_database, - DIR_LIST_DATABASES, NULL, show_hidden_files); - if (rdb_list && rdb_list->size) { size_t i; @@ -786,9 +899,6 @@ enum manual_content_scan_playlist_refresh_status } } - string_list_free(rdb_list); -#endif - /* > If system name does not match a database * file, then check whether it matches the * content directory name */ @@ -853,7 +963,7 @@ enum manual_content_scan_playlist_refresh_status fill_pathname_expand_special(scan_dat_file_path, dat_file_path, sizeof(scan_dat_file_path)); - switch (manual_content_scan_validate_dat_file_path()) + switch (manual_content_scan_validate_dat(fs ? &fs->dat_size : NULL)) { case MANUAL_CONTENT_SCAN_DAT_FILE_INVALID: return MANUAL_CONTENT_SCAN_PLAYLIST_REFRESH_INVALID_DAT_FILE; @@ -880,6 +990,466 @@ enum manual_content_scan_playlist_refresh_status return MANUAL_CONTENT_SCAN_PLAYLIST_REFRESH_OK; } +enum manual_content_scan_playlist_refresh_status + manual_content_scan_set_menu_from_playlist(playlist_t *playlist, + const char *path_content_database, bool show_hidden_files) +{ + enum manual_content_scan_playlist_refresh_status status; + struct string_list *rdb_list = NULL; +#ifdef HAVE_LIBRETRODB + /* > If platform has database support, get names + * of all installed database files */ + rdb_list = dir_list_new_special( + path_content_database, + DIR_LIST_DATABASES, NULL, show_hidden_files); +#endif + status = manual_content_scan_set_menu_from_playlist_list(playlist, + rdb_list); + string_list_free(rdb_list); + return status; +} + +/* Whether @name is an installed database: the basename, + * without extension, of a path in @rdb_list */ +static bool manual_content_scan_rdb_list_has( + const struct string_list *rdb_list, const char *name) +{ + size_t i; + + if (!rdb_list) + return false; + + for (i = 0; i < rdb_list->size; i++) + { + char rdb_name[NAME_MAX_LENGTH]; + const char *rdb_path = rdb_list->elems[i].data; + if (!rdb_path || !*rdb_path) + continue; + fill_pathname(rdb_name, path_basename(rdb_path), "", + sizeof(rdb_name)); + if (string_is_equal(name, rdb_name)) + return true; + } + + return false; +} + +enum manual_content_scan_core_key +{ + MANUAL_CONTENT_SCAN_CORE_KEY_PATH = 0, + MANUAL_CONTENT_SCAN_CORE_KEY_FILE_ID_EXACT, + MANUAL_CONTENT_SCAN_CORE_KEY_FILE_ID, + MANUAL_CONTENT_SCAN_CORE_KEY_DISPLAY_NAME +}; + +/* Counts the cores whose @key is @s; *found gets the first */ +static unsigned manual_content_scan_match_cores( + core_info_list_t *core_info_list, + enum manual_content_scan_core_key key, const char *s, + const core_info_t **found) +{ + size_t i; + unsigned count = 0; +#ifdef _WIN32 + /* Either slash: core_info joins its folder and file with '/' */ + char *core_path = NULL; + + if ( key == MANUAL_CONTENT_SCAN_CORE_KEY_PATH + && !(core_path = (char*)malloc(PATH_MAX_LENGTH))) + return 0; +#endif + + for (i = 0; i < core_info_list->count; i++) + { + const core_info_t *core_info = core_info_get(core_info_list, i); + bool match = false; + + if (!core_info) + continue; + + switch (key) + { + case MANUAL_CONTENT_SCAN_CORE_KEY_PATH: +#ifdef _WIN32 + if (core_info->path) + { + strlcpy(core_path, core_info->path, PATH_MAX_LENGTH); + pathname_conform_slashes_to_os(core_path); + match = string_is_equal_case_insensitive(core_path, s); + } +#else + match = string_is_equal(core_info->path, s); +#endif + break; + case MANUAL_CONTENT_SCAN_CORE_KEY_FILE_ID_EXACT: + match = string_is_equal(core_info->core_file_id.str, s); + break; + case MANUAL_CONTENT_SCAN_CORE_KEY_FILE_ID: + match = string_is_equal_case_insensitive( + core_info->core_file_id.str, s); + break; + case MANUAL_CONTENT_SCAN_CORE_KEY_DISPLAY_NAME: + match = string_is_equal(core_info->display_name, s); + break; + } + + if (match && !count++) + *found = core_info; + } + +#ifdef _WIN32 + free(core_path); +#endif + return count; +} + +/* Whether @core is written as a path: display names hold + * slashes too, so a path that matches no core falls through + * to the names */ +static bool manual_content_scan_core_is_path(const char *core) +{ + return strchr(core, '/') +#ifdef _WIN32 + || strchr(core, '\\') +#endif + || *core == '~'; +} + +/* @core, a path, as the core list holds paths: expanded, and + * "//" and ".." folded in an absolute path too; "" when it + * resolves to nothing. Asks the file system. */ +static void manual_content_scan_core_path_resolve(char *s, + const char *core, size_t len) +{ + fill_pathname_expand_special(s, core, len); +#ifndef RARCH_CONSOLE + if ( *s + && !path_resolve_realpath(s, len, false) + && !path_is_absolute(s)) + *s = '\0'; +#endif +#ifdef _WIN32 + pathname_conform_slashes_to_os(s); +#endif +} + +/* An override DAT path as the scan opens it: expanded, and a + * relative one resolved; "" when it resolves to nothing. + * Asks the file system. */ +static void manual_content_scan_dat_path_resolve(char *s, + const char *dat, size_t len) +{ + fill_pathname_expand_special(s, dat, len); +#ifndef RARCH_CONSOLE + if ( *s && !path_is_absolute(s) + && !path_resolve_realpath(s, len, false)) + *s = '\0'; +#endif +} + +/* Finds the core @core names: a path, else a file name or + * file id, else a display name. The first of these that + * matches decides, and *count gets its matches. + * > @resolved: @core resolved as a path already, or NULL to + * resolve it here */ +static const core_info_t *manual_content_scan_find_core( + const char *core, const char *resolved, unsigned *count) +{ + core_info_list_t *core_info_list = NULL; + const core_info_t *found = NULL; + char *s = NULL; + char *lower = NULL; + + *count = 0; + core_info_get_list(&core_info_list); + + if (!core_info_list || !(s = (char*)malloc(PATH_MAX_LENGTH))) + return NULL; + + if (manual_content_scan_core_is_path(core)) + { + if (resolved) + strlcpy(s, resolved, PATH_MAX_LENGTH); + else + manual_content_scan_core_path_resolve(s, core, PATH_MAX_LENGTH); + if (*s) + *count = manual_content_scan_match_cores(core_info_list, + MANUAL_CONTENT_SCAN_CORE_KEY_PATH, s, &found); + } + + /* Lower case, as core_info finds "_libretro" in lower case only */ + if (!*count && (lower = strdup(core))) + { + size_t i, _len; + string_to_lower(lower); + /* core_info's rule would cut a bare id's last "_" part */ + if (strstr(lower, "_libretro")) + _len = core_info_get_file_id(lower, s, PATH_MAX_LENGTH); + else + { + _len = fill_pathname(s, lower, "", PATH_MAX_LENGTH); + strlcpy(s + _len, "_libretro", PATH_MAX_LENGTH - _len); + } + /* The case typed, for an exact match first */ + for (i = 0; i < _len; i++) + if (s[i] == lower[i]) + s[i] = core[i]; + free(lower); + *count = manual_content_scan_match_cores(core_info_list, + MANUAL_CONTENT_SCAN_CORE_KEY_FILE_ID_EXACT, s, &found); + if (!*count) + *count = manual_content_scan_match_cores(core_info_list, + MANUAL_CONTENT_SCAN_CORE_KEY_FILE_ID, s, &found); + } + + if (!*count) + *count = manual_content_scan_match_cores(core_info_list, + MANUAL_CONTENT_SCAN_CORE_KEY_DISPLAY_NAME, core, &found); + + free(s); + return found; +} + +manual_content_scan_overrides_t *manual_content_scan_overrides_new(void) +{ + return (manual_content_scan_overrides_t*)calloc(1, + sizeof(manual_content_scan_overrides_t)); +} + +void manual_content_scan_overrides_free(manual_content_scan_overrides_t *ov) +{ + if (!ov) + return; + free(ov->playlist); + free(ov->database); + free(ov->dat); + free(ov->core); + free(ov->exts); + free(ov); +} + +manual_content_scan_fs_t *manual_content_scan_fs_read( + const char *content_dir, const char *dat, + const char *ov_core, const char *ov_dat) +{ + manual_content_scan_fs_t *fs = (manual_content_scan_fs_t*)calloc(1, + sizeof(*fs)); + char *s = (char*)malloc(PATH_MAX_LENGTH); + + if (!fs || !s) + goto error; + + /* The folder as manual_content_scan_set_menu_content_dir() + * stats it: expanded, one trailing slash gone */ + if (content_dir && *content_dir) + { + size_t _len = fill_pathname_expand_special(s, content_dir, + PATH_MAX_LENGTH); + if (_len > 0 && _len < PATH_MAX_LENGTH + && s[_len - 1] == PATH_DEFAULT_SLASH_C()) + s[_len - 1] = '\0'; + if (*s) + fs->content_stat = path_stat(s); + } + + /* The saved DAT, as a refresh expands it */ + fs->dat_size = -1; + if (dat && *dat) + { + fill_pathname_expand_special(s, dat, PATH_MAX_LENGTH); + if (*s && path_is_valid(s)) + fs->dat_size = path_get_size(s); + } + + fs->ov_dat_size = -1; + if (ov_dat) + { + manual_content_scan_dat_path_resolve(s, ov_dat, PATH_MAX_LENGTH); + if (!(fs->ov_dat = strdup(s))) + goto error; + if (*s && path_is_valid(s)) + fs->ov_dat_size = path_get_size(s); + } + + if (ov_core && manual_content_scan_core_is_path(ov_core)) + { + manual_content_scan_core_path_resolve(s, ov_core, PATH_MAX_LENGTH); + if (!(fs->ov_core = strdup(s))) + goto error; + } + + free(s); + return fs; + +error: + free(s); + manual_content_scan_fs_free(fs); + return NULL; +} + +void manual_content_scan_fs_free(manual_content_scan_fs_t *fs) +{ + if (!fs) + return; + free(fs->ov_dat); + free(fs->ov_core); + free(fs); +} + +/* Lays @ov over the current settings and sets the Custom + * method, stopping at the first override that does not + * resolve; NULL @ov changes nothing. + * > @rdb_list: paths of the installed databases, or NULL + * > @count: the cores @ov->core matched, when not NULL */ +enum manual_content_scan_override_status + manual_content_scan_apply_overrides( + const manual_content_scan_overrides_t *ov, + const struct string_list *rdb_list, unsigned *count) +{ + return manual_content_scan_apply_overrides_fs(ov, rdb_list, count, + NULL); +} + +enum manual_content_scan_override_status + manual_content_scan_apply_overrides_fs( + const manual_content_scan_overrides_t *ov, + const struct string_list *rdb_list, unsigned *count, + const manual_content_scan_fs_t *fs) +{ + if (count) + *count = 0; + + if (!ov) + return MANUAL_CONTENT_SCAN_OVR_OK; + + if (ov->core) + { + unsigned matches = 0; + const core_info_t *core_info = manual_content_scan_find_core( + ov->core, fs ? fs->ov_core : NULL, &matches); + + if (count) + *count = matches; + if (matches > 1) + return MANUAL_CONTENT_SCAN_OVR_AMBIGUOUS_CORE; + if (!core_info || !manual_content_scan_set_core(core_info)) + return MANUAL_CONTENT_SCAN_OVR_NO_SUCH_CORE; + scan_settings.core_type = MANUAL_CONTENT_SCAN_CORE_SET; + } + + if (ov->database) + { + if (!manual_content_scan_rdb_list_has(rdb_list, ov->database)) + return MANUAL_CONTENT_SCAN_OVR_NO_SUCH_DATABASE; + /* A refreshed database playlist's name shares this + * buffer: it keeps that name as a custom one */ + if ( scan_settings.system_name_type + == MANUAL_CONTENT_SCAN_SYSTEM_NAME_DATABASE + && !string_is_equal(scan_system_name_database, ov->database)) + { + strlcpy(scan_system_name_custom, scan_system_name_database, + sizeof(scan_system_name_custom)); + manual_content_scan_set_menu_system_name( + MANUAL_CONTENT_SCAN_SYSTEM_NAME_CUSTOM, NULL); + } + manual_content_scan_set_menu_scan_db_select( + MANUAL_CONTENT_SCAN_SELECT_DB_SPECIFIC, ov->database); + } + else if (ov->flags & MANUAL_CONTENT_SCAN_OVR_FIRST_DB) + manual_content_scan_set_menu_scan_db_select( + MANUAL_CONTENT_SCAN_SELECT_DB_AUTO_FIRST_MATCH, NULL); + + /* A database playlist's name and the database to match + * share one buffer: next to another database, the + * playlist is custom */ + if (ov->playlist) + { + if ( manual_content_scan_rdb_list_has(rdb_list, ov->playlist) + && (!ov->database || string_is_equal(ov->database, ov->playlist))) + manual_content_scan_set_menu_system_name( + MANUAL_CONTENT_SCAN_SYSTEM_NAME_DATABASE, ov->playlist); + else + { + strlcpy(scan_system_name_custom, ov->playlist, + sizeof(scan_system_name_custom)); + manual_content_scan_scrub_system_name_custom(); + manual_content_scan_set_menu_system_name( + MANUAL_CONTENT_SCAN_SYSTEM_NAME_CUSTOM, NULL); + } + } + + if (ov->dat) + { + if ( (ov->flags & MANUAL_CONTENT_SCAN_OVR_MATCH) + && ov->match == MANUAL_CONTENT_SCAN_USE_DB_NONE) + return MANUAL_CONTENT_SCAN_OVR_DAT_WITHOUT_MATCH; + + if (fs && fs->ov_dat) + strlcpy(scan_dat_file_path, fs->ov_dat, + sizeof(scan_dat_file_path)); + else + manual_content_scan_dat_path_resolve(scan_dat_file_path, ov->dat, + sizeof(scan_dat_file_path)); + switch (manual_content_scan_validate_dat( + (fs && fs->ov_dat) ? &fs->ov_dat_size : NULL)) + { + case MANUAL_CONTENT_SCAN_DAT_FILE_OK: + break; + case MANUAL_CONTENT_SCAN_DAT_FILE_TOO_LARGE: + return MANUAL_CONTENT_SCAN_OVR_DAT_TOO_LARGE; + default: + return MANUAL_CONTENT_SCAN_OVR_NOT_A_DAT; + } + + manual_content_scan_set_menu_scan_use_db( + ( (ov->flags & MANUAL_CONTENT_SCAN_OVR_MATCH) + && ov->match == MANUAL_CONTENT_SCAN_USE_DB_LOOSE) + ? MANUAL_CONTENT_SCAN_USE_DB_DAT_LOOSE + : MANUAL_CONTENT_SCAN_USE_DB_DAT_STRICT); + } + else if (ov->flags & MANUAL_CONTENT_SCAN_OVR_MATCH) + { + enum manual_content_scan_db_usage usage = ov->match; + + /* A refresh keeps its saved DAT unless the match is none */ + if ( scan_settings.db_usage == MANUAL_CONTENT_SCAN_USE_DB_DAT_STRICT + || scan_settings.db_usage == MANUAL_CONTENT_SCAN_USE_DB_DAT_LOOSE) + { + if (ov->match == MANUAL_CONTENT_SCAN_USE_DB_STRICT) + usage = MANUAL_CONTENT_SCAN_USE_DB_DAT_STRICT; + else if (ov->match == MANUAL_CONTENT_SCAN_USE_DB_LOOSE) + usage = MANUAL_CONTENT_SCAN_USE_DB_DAT_LOOSE; + else + scan_dat_file_path[0] = '\0'; + } + + manual_content_scan_set_menu_scan_use_db(usage); + } + + if (ov->exts) + { + strlcpy(scan_file_exts_custom, ov->exts, + sizeof(scan_file_exts_custom)); + manual_content_scan_scrub_file_exts_custom(); + } + + if (ov->flags & MANUAL_CONTENT_SCAN_OVR_NO_RECURSIVE) + scan_settings.search_recursively = false; + if (ov->flags & MANUAL_CONTENT_SCAN_OVR_NO_ARCHIVES) + scan_settings.search_archives = false; + if (ov->flags & MANUAL_CONTENT_SCAN_OVR_OVERWRITE) + scan_settings.overwrite_playlist = true; + if (ov->flags & MANUAL_CONTENT_SCAN_OVR_VALIDATE) + scan_settings.validate_entries = true; + if (ov->flags & MANUAL_CONTENT_SCAN_OVR_OMIT_DB_REF) + scan_settings.omit_db_ref = true; + + manual_content_scan_set_menu_scan_method( + MANUAL_CONTENT_SCAN_METHOD_CUSTOM); + + return MANUAL_CONTENT_SCAN_OVR_OK; +} + /* Menu getters */ /* Fetches content directory for next manual scan @@ -1340,6 +1910,32 @@ struct string_list *manual_content_scan_get_menu_core_name_list(void) /* Task Helpers */ /****************/ +/* The system name type a scan of the current settings takes: an + * automatic name is filled when the settings predict it. */ +static enum manual_content_scan_system_name_type + manual_content_scan_task_name_type(void) +{ + if (scan_settings.system_name_type == MANUAL_CONTENT_SCAN_SYSTEM_NAME_AUTO) + { + if (scan_settings.db_usage == MANUAL_CONTENT_SCAN_USE_DB_DAT_STRICT || + scan_settings.db_usage == MANUAL_CONTENT_SCAN_USE_DB_DAT_LOOSE || + scan_settings.db_usage == MANUAL_CONTENT_SCAN_USE_DB_NONE) + return MANUAL_CONTENT_SCAN_SYSTEM_NAME_CONTENT_DIR; + if (scan_settings.db_selection == MANUAL_CONTENT_SCAN_SELECT_DB_SPECIFIC) + return MANUAL_CONTENT_SCAN_SYSTEM_NAME_DATABASE; + } + return scan_settings.system_name_type; +} + +/* True if a scan of the current settings writes one playlist + * (target_is_single_determined_playlist). */ +bool manual_content_scan_is_single_playlist(void) +{ + return scan_settings.scan_method != MANUAL_CONTENT_SCAN_METHOD_AUTOMATIC + && manual_content_scan_task_name_type() + != MANUAL_CONTENT_SCAN_SYSTEM_NAME_AUTO; +} + /* Parses current manual content scan settings, * and extracts all information required to configure * a manual content scan task. @@ -1386,15 +1982,7 @@ bool manual_content_scan_get_task_config( sizeof(task_config->content_dir)); /* If name is set as automatic, but is predictable, fill it now. */ - if (scan_settings.system_name_type == MANUAL_CONTENT_SCAN_SYSTEM_NAME_AUTO) - { - if (scan_settings.db_usage == MANUAL_CONTENT_SCAN_USE_DB_DAT_STRICT || - scan_settings.db_usage == MANUAL_CONTENT_SCAN_USE_DB_DAT_LOOSE || - scan_settings.db_usage == MANUAL_CONTENT_SCAN_USE_DB_NONE) - scan_settings.system_name_type = MANUAL_CONTENT_SCAN_SYSTEM_NAME_CONTENT_DIR; - else if (scan_settings.db_selection == MANUAL_CONTENT_SCAN_SELECT_DB_SPECIFIC) - scan_settings.system_name_type = MANUAL_CONTENT_SCAN_SYSTEM_NAME_DATABASE; - } + scan_settings.system_name_type = manual_content_scan_task_name_type(); /* Get target playlist ("system name") * * The system name is not a leaf value: the scan pipeline appends a diff --git a/manual_content_scan.h b/manual_content_scan.h index 8b6177129314..44aa929cd861 100644 --- a/manual_content_scan.h +++ b/manual_content_scan.h @@ -104,6 +104,43 @@ enum manual_content_scan_playlist_refresh_status MANUAL_CONTENT_SCAN_PLAYLIST_REFRESH_DAT_FILE_TOO_LARGE }; +/* Defines all possible return values for + * manual_content_scan_apply_overrides() */ +enum manual_content_scan_override_status +{ + MANUAL_CONTENT_SCAN_OVR_OK = 0, + MANUAL_CONTENT_SCAN_OVR_NO_SUCH_CORE, + MANUAL_CONTENT_SCAN_OVR_AMBIGUOUS_CORE, + MANUAL_CONTENT_SCAN_OVR_NO_SUCH_DATABASE, + MANUAL_CONTENT_SCAN_OVR_NOT_A_DAT, + MANUAL_CONTENT_SCAN_OVR_DAT_TOO_LARGE, + MANUAL_CONTENT_SCAN_OVR_DAT_WITHOUT_MATCH +}; + +enum manual_content_scan_override_flags +{ + MANUAL_CONTENT_SCAN_OVR_MATCH = (1 << 0), /* 'match' is set */ + MANUAL_CONTENT_SCAN_OVR_FIRST_DB = (1 << 1), + MANUAL_CONTENT_SCAN_OVR_NO_RECURSIVE = (1 << 2), + MANUAL_CONTENT_SCAN_OVR_NO_ARCHIVES = (1 << 3), + MANUAL_CONTENT_SCAN_OVR_OVERWRITE = (1 << 4), + MANUAL_CONTENT_SCAN_OVR_VALIDATE = (1 << 5), + MANUAL_CONTENT_SCAN_OVR_OMIT_DB_REF = (1 << 6) +}; + +/* Settings laid over a scan's own; a NULL string or a clear + * flag leaves its setting as it is */ +typedef struct manual_content_scan_overrides +{ + char *playlist; /* --scan-playlist */ + char *database; /* --scan-database=NAME */ + char *dat; /* --scan-dat, as typed */ + char *core; /* --scan-core, as typed */ + char *exts; /* --scan-exts, space separated */ + enum manual_content_scan_db_usage match; /* STRICT, LOOSE or NONE */ + uint16_t flags; /* MANUAL_CONTENT_SCAN_OVR_* */ +} manual_content_scan_overrides_t; + /* Holds all configuration parameters required * for a manual content scan task */ typedef struct @@ -223,11 +260,47 @@ enum manual_content_scan_dat_file_path_status /* Menu setters */ +/* Sets every setting back to its default */ +void manual_content_scan_reset_settings(void); + +/* The Manual Scan settings as they are, for a scan chain to put back + * once it has run its own: NULL out of memory. Restoring frees it. */ +typedef struct manual_content_scan_saved manual_content_scan_saved_t; +manual_content_scan_saved_t *manual_content_scan_save_settings(void); +void manual_content_scan_restore_settings(manual_content_scan_saved_t *saved); + /* Sets content directory for next manual scan * operation. * Returns true if content directory is valid. */ bool manual_content_scan_set_menu_content_dir(const char *content_dir); +/* What the file system says of one scan's paths, read off the main + * thread by manual_content_scan_fs_read(), so that the _fs setters + * below ask it nothing. Only the paths it was read for: it holds no + * reference to the settings. */ +typedef struct manual_content_scan_fs +{ + char *ov_dat; /* @ov_dat expanded and resolved, or NULL */ + char *ov_core; /* @ov_core resolved when it is a path, or NULL */ + int64_t dat_size; /* the saved DAT's size; <= 0: not there */ + int64_t ov_dat_size; /* likewise, @ov_dat's */ + int content_stat; /* path_stat() of the content folder */ +} manual_content_scan_fs_t; + +/* Any thread: reads nothing but its arguments and the file system. + * @content_dir and @dat as the settings hold them, before expansion; + * @ov_core and @ov_dat an item's overrides, each NULL when unset. + * NULL out of memory. Free with manual_content_scan_fs_free(). */ +manual_content_scan_fs_t *manual_content_scan_fs_read( + const char *content_dir, const char *dat, + const char *ov_core, const char *ov_dat); +void manual_content_scan_fs_free(manual_content_scan_fs_t *fs); + +/* As manual_content_scan_set_menu_content_dir(), the folder's stat + * taken from @fs (NULL: asked now) */ +bool manual_content_scan_set_menu_content_dir_fs(const char *content_dir, + const manual_content_scan_fs_t *fs); + bool manual_content_scan_set_menu_scan_method( enum manual_content_scan_method method); @@ -269,6 +342,40 @@ enum manual_content_scan_playlist_refresh_status manual_content_scan_set_menu_from_playlist(playlist_t *playlist, const char *path_content_database, bool show_hidden_files); +/* As manual_content_scan_set_menu_from_playlist(), with the + * paths of the installed databases in @rdb_list (NULL: none). */ +enum manual_content_scan_playlist_refresh_status + manual_content_scan_set_menu_from_playlist_list(playlist_t *playlist, + const struct string_list *rdb_list); + +/* As manual_content_scan_set_menu_from_playlist_list(), the folder + * and the DAT as @fs read them (NULL: asked now) */ +enum manual_content_scan_playlist_refresh_status + manual_content_scan_set_menu_from_playlist_list_fs( + playlist_t *playlist, const struct string_list *rdb_list, + const manual_content_scan_fs_t *fs); + +manual_content_scan_overrides_t *manual_content_scan_overrides_new(void); +void manual_content_scan_overrides_free(manual_content_scan_overrides_t *ov); + +/* Lays @ov over the current settings and sets the Custom + * method, stopping at the first override that does not + * resolve; NULL @ov changes nothing. + * > @rdb_list: paths of the installed databases, or NULL + * > @count: the cores @ov->core matched, when not NULL */ +enum manual_content_scan_override_status + manual_content_scan_apply_overrides( + const manual_content_scan_overrides_t *ov, + const struct string_list *rdb_list, unsigned *count); + +/* As manual_content_scan_apply_overrides(), the core path and the + * DAT as @fs read them for @ov (NULL: asked now) */ +enum manual_content_scan_override_status + manual_content_scan_apply_overrides_fs( + const manual_content_scan_overrides_t *ov, + const struct string_list *rdb_list, unsigned *count, + const manual_content_scan_fs_t *fs); + /* Menu getters */ /* Fetches content directory for next manual scan @@ -318,6 +425,10 @@ struct string_list *manual_content_scan_get_menu_core_name_list(void); /* Task Helpers */ /****************/ +/* True if a scan of the current settings writes one playlist + * (target_is_single_determined_playlist). */ +bool manual_content_scan_is_single_playlist(void); + /* Parses current manual content scan settings, * and extracts all information required to configure * a manual content scan task. diff --git a/menu/cbs/menu_cbs_ok.c b/menu/cbs/menu_cbs_ok.c index 53a916a5b9e4..7ca487568c46 100644 --- a/menu/cbs/menu_cbs_ok.c +++ b/menu/cbs/menu_cbs_ok.c @@ -9123,6 +9123,8 @@ static int action_ok_manual_content_scan_start(const char *path, { playlist_config_t playlist_config; settings_t *settings = config_get_ptr(); + if (task_dbscan_chain_refuse()) + return 0; /* Note: playlist_config.path will set by the * task itself */ playlist_config.capacity = COLLECTION_SIZE; @@ -9133,7 +9135,7 @@ static int action_ok_manual_content_scan_start(const char *path, settings->bools.playlist_portable_paths ? settings->paths.directory_menu_content : NULL); - task_push_manual_content_scan(true, NULL); + task_push_manual_content_scan(true, NULL, NULL); return 0; } @@ -9599,6 +9601,8 @@ static int action_ok_playlist_refresh(const char *path, playlist_config = playlist_get_config(playlist); if (!playlist_config || !*playlist_config->path) return -1; + if (task_dbscan_chain_refuse()) + return 0; stat = manual_content_scan_set_menu_from_playlist(playlist, settings->paths.path_content_database, settings->bools.show_hidden_files); @@ -9683,7 +9687,35 @@ static int action_ok_playlist_refresh(const char *path, settings->bools.playlist_portable_paths ? settings->paths.directory_menu_content : NULL); - task_push_manual_content_scan(false, NULL); + task_push_manual_content_scan(false, NULL, NULL); + } + return 0; +} + +static void action_ok_playlist_refresh_all_cb( + const task_dbscan_chain_result_t *res, void *user_data) +{ + char msg[128]; + int n = snprintf(msg, sizeof(msg), + msg_hash_to_str(MSG_PLAYLIST_MANAGER_REFRESH_ALL_DONE), + res->ran, res->skipped, res->failed); + /* the length written, not the one a translation asked for */ + size_t _len = n < 0 ? 0 : (size_t)n >= sizeof(msg) ? sizeof(msg) - 1 : (size_t)n; + runloop_msg_queue_push(msg, _len, 1, 180, true, NULL, + MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_INFO); +} + +static int action_ok_playlist_refresh_all(const char *path, + const char *label, unsigned type, size_t idx, size_t entry_idx) +{ + task_dbscan_item_t *item = NULL; + if (task_dbscan_chain_refuse_start()) + return 0; + if ((item = (task_dbscan_item_t*)calloc(1, sizeof(*item)))) + { + item->kind = TASK_DBSCAN_ITEM_REFRESH_ALL; + task_push_dbscan_chain(item, 1, NULL, + action_ok_playlist_refresh_all_cb, NULL); } return 0; } @@ -10031,6 +10063,7 @@ static int menu_cbs_init_bind_ok_compare_label(menu_file_list_cbs_t *cbs, {MENU_ENUM_LABEL_PLAYLIST_MANAGER_RESET_CORES, action_ok_playlist_reset_cores}, {MENU_ENUM_LABEL_PLAYLIST_MANAGER_CLEAN_PLAYLIST, action_ok_playlist_clean}, {MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_PLAYLIST, action_ok_playlist_refresh}, + {MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_ALL_PLAYLISTS, action_ok_playlist_refresh_all}, {MENU_ENUM_LABEL_ACCOUNTS_RETRO_ACHIEVEMENTS, action_ok_push_accounts_cheevos_list}, #ifdef HAVE_LAKKA {MENU_ENUM_LABEL_EJECT_DISC, action_ok_push_eject_disc}, diff --git a/menu/cbs/menu_cbs_scan.c b/menu/cbs/menu_cbs_scan.c index 565346fd2463..517d1a60b0c9 100644 --- a/menu/cbs/menu_cbs_scan.c +++ b/menu/cbs/menu_cbs_scan.c @@ -69,7 +69,7 @@ int action_scan_file(const char *path, fill_pathname_join_special(fullpath, menu_path, path, sizeof(fullpath)); - task_push_dbscan(fullpath, handle_dbscan_finished); + task_push_dbscan(fullpath, handle_dbscan_finished, NULL); return 0; } @@ -96,7 +96,7 @@ int action_scan_directory(const char *path, else strlcpy(fullpath, menu_path, sizeof(fullpath)); - task_push_dbscan(fullpath, handle_dbscan_finished); + task_push_dbscan(fullpath, handle_dbscan_finished, NULL); return 0; } diff --git a/menu/cbs/menu_cbs_sublabel.c b/menu/cbs/menu_cbs_sublabel.c index 618865afb28e..0dded411347e 100644 --- a/menu/cbs/menu_cbs_sublabel.c +++ b/menu/cbs/menu_cbs_sublabel.c @@ -2755,6 +2755,7 @@ int menu_cbs_init_bind_sublabel(menu_file_list_cbs_t *cbs, { MENU_ENUM_LABEL_PLAYLIST_MANAGER_SORT_MODE, MENU_ENUM_SUBLABEL_PLAYLIST_MANAGER_SORT_MODE }, { MENU_ENUM_LABEL_PLAYLIST_MANAGER_CLEAN_PLAYLIST, MENU_ENUM_SUBLABEL_PLAYLIST_MANAGER_CLEAN_PLAYLIST }, { MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_PLAYLIST, MENU_ENUM_SUBLABEL_PLAYLIST_MANAGER_REFRESH_PLAYLIST }, + { MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_ALL_PLAYLISTS, MENU_ENUM_SUBLABEL_PLAYLIST_MANAGER_REFRESH_ALL_PLAYLISTS }, { MENU_ENUM_LABEL_SAVE_AS_CONFIG, MENU_ENUM_SUBLABEL_SAVE_AS_CONFIG }, { MENU_ENUM_LABEL_SAVE_MAIN_CONFIG, MENU_ENUM_SUBLABEL_SAVE_MAIN_CONFIG }, { MENU_ENUM_LABEL_ONSCREEN_DISPLAY_SETTINGS, MENU_ENUM_SUBLABEL_ONSCREEN_DISPLAY_SETTINGS }, diff --git a/menu/drivers/materialui.c b/menu/drivers/materialui.c index ff1701f9a720..78d8ebef72b5 100644 --- a/menu/drivers/materialui.c +++ b/menu/drivers/materialui.c @@ -12746,6 +12746,7 @@ static void materialui_list_insert(void *userdata, || string_is_equal(label, MENU_ENUM_LABEL_UPDATE_GLSL_SHADERS_STR) || string_is_equal(label, MENU_ENUM_LABEL_UPDATE_SLANG_SHADERS_STR) || string_is_equal(label, MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_PLAYLIST_STR) + || string_is_equal(label, MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_ALL_PLAYLISTS_STR) ) { node->icon_texture_index = MUI_TEXTURE_UPDATER; diff --git a/menu/drivers/ozone.c b/menu/drivers/ozone.c index 642c96f4ff9d..b18aa113dfe1 100644 --- a/menu/drivers/ozone.c +++ b/menu/drivers/ozone.c @@ -2460,6 +2460,7 @@ static uintptr_t ozone_entries_icon_get_texture( case MENU_ENUM_LABEL_FRAME_TIME_COUNTER_SETTINGS: case MENU_ENUM_LABEL_PLAYLIST_MANAGER_CLEAN_PLAYLIST: case MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_PLAYLIST: + case MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_ALL_PLAYLISTS: case MENU_ENUM_LABEL_CLOUD_SYNC_SETTINGS: return icons_tex[OZONE_ENTRIES_ICONS_TEXTURE_RELOAD]; case MENU_ENUM_LABEL_SHUTDOWN: diff --git a/menu/drivers/xmb.c b/menu/drivers/xmb.c index 2e977e6c1316..e2dd1ab32bbb 100644 --- a/menu/drivers/xmb.c +++ b/menu/drivers/xmb.c @@ -4393,6 +4393,7 @@ static uintptr_t xmb_icon_get_id(xmb_handle_t *xmb, case MENU_ENUM_LABEL_FRAME_TIME_COUNTER_SETTINGS: case MENU_ENUM_LABEL_PLAYLIST_MANAGER_CLEAN_PLAYLIST: case MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_PLAYLIST: + case MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_ALL_PLAYLISTS: case MENU_ENUM_LABEL_CLOUD_SYNC_SETTINGS: return GFX_SURFACE_HANDLE(xmb->textures.list[XMB_TEXTURE_RELOAD]); case MENU_ENUM_LABEL_VRR_RUNLOOP_ENABLE: diff --git a/menu/menu_displaylist.c b/menu/menu_displaylist.c index c5febc3d8809..03bd693c83e7 100644 --- a/menu/menu_displaylist.c +++ b/menu/menu_displaylist.c @@ -6236,6 +6236,17 @@ static unsigned menu_displaylist_parse_playlist_manager_list( count++; #endif + /* None keeps the "No entries" fallback */ + if (count) + { + menu_entries_prepend(list, + msg_hash_to_str(MENU_ENUM_LABEL_VALUE_PLAYLIST_MANAGER_REFRESH_ALL_PLAYLISTS), + MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_ALL_PLAYLISTS_STR, + MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_ALL_PLAYLISTS, + MENU_SETTING_ACTION, 0, 0); + count++; + } + return count; } diff --git a/msg_hash.h b/msg_hash.h index 2dfe9347030d..0c4cc47dd8f6 100644 --- a/msg_hash.h +++ b/msg_hash.h @@ -29962,6 +29962,9 @@ enum msg_hash_enums MSG_PLAYLIST_MANAGER_REFRESH_INVALID_DAT_FILE, MSG_PLAYLIST_MANAGER_REFRESH_DAT_FILE_TOO_LARGE, + MENU_LABEL(PLAYLIST_MANAGER_REFRESH_ALL_PLAYLISTS), + MSG_PLAYLIST_MANAGER_REFRESH_ALL_DONE, + MENU_LABEL(CORE_UPDATER_SETTINGS), MENU_LABEL(SAVE_AS_CONFIG), MENU_LABEL(SAVE_MAIN_CONFIG), @@ -34351,6 +34354,7 @@ enum msg_hash_enums MSG_MANUAL_CONTENT_SCAN_IN_PROGRESS, MSG_MANUAL_CONTENT_SCAN_M3U_CLEANUP, MSG_MANUAL_CONTENT_SCAN_END, + MSG_MANUAL_CONTENT_SCAN_ALREADY_RUNNING, MSG_3DS_BOTTOM_MENU_DEFAULT, MSG_3DS_BOTTOM_MENU_ASSET_NOT_FOUND, diff --git a/msg_hash_lbl_str.h b/msg_hash_lbl_str.h index 3d46daa8ba41..214bda861d80 100644 --- a/msg_hash_lbl_str.h +++ b/msg_hash_lbl_str.h @@ -486,6 +486,7 @@ extern const struct msg_hash_us_lbl_blob_s msg_hash_us_lbl_blob; #define MENU_ENUM_LABEL_PLAYLIST_MANAGER_SORT_MODE_STR (msg_hash_us_lbl_blob.m_MENU_ENUM_LABEL_PLAYLIST_MANAGER_SORT_MODE) #define MENU_ENUM_LABEL_PLAYLIST_MANAGER_CLEAN_PLAYLIST_STR (msg_hash_us_lbl_blob.m_MENU_ENUM_LABEL_PLAYLIST_MANAGER_CLEAN_PLAYLIST) #define MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_PLAYLIST_STR (msg_hash_us_lbl_blob.m_MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_PLAYLIST) +#define MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_ALL_PLAYLISTS_STR (msg_hash_us_lbl_blob.m_MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_ALL_PLAYLISTS) #define MENU_ENUM_LABEL_PLAYLIST_SETTINGS_BEGIN_STR (msg_hash_us_lbl_blob.m_MENU_ENUM_LABEL_PLAYLIST_SETTINGS_BEGIN) #define MENU_ENUM_LABEL_RDB_ENTRY_STR (msg_hash_us_lbl_blob.m_MENU_ENUM_LABEL_RDB_ENTRY) #define MENU_ENUM_LABEL_RDB_ENTRY_DETAIL_STR (msg_hash_us_lbl_blob.m_MENU_ENUM_LABEL_RDB_ENTRY_DETAIL) @@ -2238,6 +2239,7 @@ extern const struct msg_hash_us_lbl_blob_s msg_hash_us_lbl_blob; #define MENU_ENUM_LABEL_PLAYLIST_MANAGER_SORT_MODE_STR "playlist_manager_sort_mode" #define MENU_ENUM_LABEL_PLAYLIST_MANAGER_CLEAN_PLAYLIST_STR "playlist_manager_clean_playlist" #define MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_PLAYLIST_STR "playlist_manager_refresh_playlist" +#define MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_ALL_PLAYLISTS_STR "playlist_manager_refresh_all_playlists" #define MENU_ENUM_LABEL_PLAYLIST_SETTINGS_BEGIN_STR "playlist_settings_begin" #define MENU_ENUM_LABEL_RDB_ENTRY_STR "rdb_entry" #define MENU_ENUM_LABEL_RDB_ENTRY_DETAIL_STR "rdb_entry_detail" diff --git a/playlist.c b/playlist.c index 5d4ece7a0d0f..4ee2797da8b6 100644 --- a/playlist.c +++ b/playlist.c @@ -32,6 +32,8 @@ #include #include #include +#include +#include #include "playlist.h" #include "verbosity.h" @@ -75,7 +77,9 @@ enum content_playlist_flags CNT_PLAYLIST_FLG_MOD = (1 << 0), CNT_PLAYLIST_FLG_OLD_FMT = (1 << 1), CNT_PLAYLIST_FLG_COMPRESSED = (1 << 2), - CNT_PLAYLIST_FLG_CACHED_EXT = (1 << 3) + CNT_PLAYLIST_FLG_CACHED_EXT = (1 << 3), + /* read by playlist_parse_begin_header(): no entries, never written */ + CNT_PLAYLIST_FLG_HEADER_ONLY = (1 << 4) }; struct content_playlist @@ -111,7 +115,9 @@ enum json_ctx_flags JSON_CTX_FLG_IN_ITEMS = (1 << 0), JSON_CTX_FLG_IN_SUBSYSTEM_CONTENT = (1 << 1), JSON_CTX_FLG_CAPACITY_EXCEEDED = (1 << 2), - JSON_CTX_FLG_OOM = (1 << 3) + JSON_CTX_FLG_OOM = (1 << 3), + JSON_CTX_FLG_HEADER_ONLY = (1 << 4), /* stop at "items" */ + JSON_CTX_FLG_HEADER_DONE = (1 << 5) }; /* The runtime and last-played values live packed inside the entry and @@ -156,9 +162,27 @@ typedef struct static playlist_t *playlist_cached = NULL; /* Set when the cached playlist's file has been rewritten by something * else, so the next request re-reads it. See - * playlist_cached_after_write for why this is a flag and not a free. */ + * playlist_cached_note_write for why this is a flag and not a free. */ static bool playlist_cached_stale = false; +/* A write reaches the cache through a note, from whatever thread made + * it: the main thread applies the notes when it next decides whether + * the cache may stand in, so no writer reads or writes playlist_cached. + * Where the pointer atomics or threads are missing the write applies + * at once, as every write then comes from the one thread. */ +#if defined(HAVE_THREADS) && defined(RETRO_ATOMIC_HAS_PTR) +#define PLAYLIST_WRITE_NOTES 1 +typedef struct playlist_write_note +{ + mpsc_stack_node_t node; + const playlist_t *written; /* compared, never dereferenced */ + int64_t size; /* the file's size after the write */ + char path[1]; /* allocated to fit */ +} playlist_write_note_t; + +static mpsc_stack_t playlist_write_notes; +#endif + typedef int (playlist_sort_fun_t)( const struct playlist_entry *a, const struct playlist_entry *b); @@ -2168,40 +2192,72 @@ void playlist_write_runtime_file(playlist_t *playlist) } } -/* A playlist file has just been written. If the cached playlist is a - * different object reading the same path, its contents are now stale - - * a scan or a playlist-manager task writing the file the menu happens - * to have open is the ordinary way this happens. Drop it rather than - * let a later reuse serve what is on disk no longer. When the cached - * playlist is the one that was written, it is still authoritative; only - * its size stamp needs to catch up. */ -static void playlist_cached_after_write(playlist_t *written) +/* What one write means for the cache: a write of the cached playlist + * itself moves its size stamp; any other write to its file leaves it + * stale, flagged rather than freed so nothing holding the pointer + * playlist_get_cached returned loses it (the next playlist_init_cached + * re-reads, at a point where nothing holds the old one). Main thread. */ +static void playlist_cached_note_write(const playlist_t *written, + const char *path, int64_t size) { - if (!playlist_cached || !written || !*written->config.path) + if (!playlist_cached || !string_is_equal(playlist_cached->config.path, path)) return; if (playlist_cached == written) + playlist_cached->file_size = size; + else + playlist_cached_stale = true; +} + +/* The notes since the last look, newest first: only the newest write of + * the cached playlist sets its stamp. Main thread. */ +static void playlist_cached_apply_writes(void) +{ +#ifdef PLAYLIST_WRITE_NOTES + bool sized = false; + mpsc_stack_node_t *node = mpsc_stack_drain(&playlist_write_notes); + while (node) { - playlist_cached->file_size = path_get_size(written->config.path); - return; + mpsc_stack_node_t *next = node->next; + playlist_write_note_t *n = (playlist_write_note_t*)node; + if (playlist_cached && n->written == playlist_cached) + { + if (!sized) + playlist_cached_note_write(n->written, n->path, n->size); + sized = true; + } + else + playlist_cached_note_write(n->written, n->path, n->size); + free(n); + node = next; } - if (string_is_equal(playlist_cached->config.path, written->config.path)) +#endif +} + +/* A playlist file has just been written, possibly by a task on a + * worker. A scan or a playlist-manager task writing the file the menu + * has open is the ordinary way the cache goes stale. */ +static void playlist_cached_after_write(playlist_t *written) +{ +#ifdef PLAYLIST_WRITE_NOTES + size_t len; + playlist_write_note_t *n; + if (!written || !*written->config.path) + return; + len = strlen(written->config.path); + if (!(n = (playlist_write_note_t*)malloc(sizeof(*n) + len))) { - /* Mark it, do not free it. - * - * This runs from whatever happened to write the file, and the - * commonest case is the history list: launching an entry pushes - * it onto g_defaults.content_history, which is a different - * playlist_t for the same path as the one the menu is displaying - * from. Freeing here would pull that object out from under - * every menu_displaylist caller still holding what - * playlist_get_cached returned - a use after free whose symptoms - * appear later and somewhere else entirely. - * - * The staleness is what matters, not the memory: flagging it - * makes the next playlist_init_cached re-read, which is a point - * where nothing is holding the old pointer. */ - playlist_cached_stale = true; + /* no note to leave: a later size check still catches a change */ + return; } + n->written = written; + n->size = path_get_size(written->config.path); + memcpy(n->path, written->config.path, len + 1); + mpsc_stack_push(&playlist_write_notes, &n->node); +#else + if (written && *written->config.path) + playlist_cached_note_write(written, written->config.path, + path_get_size(written->config.path)); +#endif } void playlist_write_file(playlist_t *playlist) @@ -2226,6 +2282,8 @@ void playlist_write_file(playlist_t *playlist) if ( !playlist || !*playlist->config.path + /* a header alone would replace the file's entries with none */ + || (playlist->flags & CNT_PLAYLIST_FLG_HEADER_ONLY) || !( (playlist->flags & CNT_PLAYLIST_FLG_MOD) #if defined(HAVE_COMPRESSION) || (pl_compressed != playlist->config.compress) @@ -3113,7 +3171,13 @@ static bool JSONObjectMemberHandler(void *context, const char *pValue, size_t le break; case 'i': if (!strcmp(pValue, "items")) - pCtx->flags |= JSON_CTX_FLG_IN_ITEMS; + { + /* the writer puts every header field before the items */ + if (pCtx->flags & JSON_CTX_FLG_HEADER_ONLY) + pCtx->flags |= JSON_CTX_FLG_HEADER_DONE; + else + pCtx->flags |= JSON_CTX_FLG_IN_ITEMS; + } break; case 'l': if (!strcmp(pValue, "label_display_mode")) @@ -3215,6 +3279,7 @@ struct playlist_parse enum playlist_parse_phase phase; bool res; bool autofix_scan_done; + bool header_only; }; static void playlist_parse_close_io(playlist_parse_t *p) @@ -3253,6 +3318,35 @@ static void playlist_parse_close_io(playlist_parse_t *p) /* Decide what follows the parse: the autofix pass when it applies, */ /* completion otherwise. Mirrors the condition playlist_init */ /* historically evaluated after playlist_read_file returned. */ +/* The scan record's paths under the base content directory @p's + * config names, as autofix moves them. */ +static void playlist_rebase_scan_record(playlist_t *playlist, + size_t oldref_len, size_t newref_len) +{ + char tmp[PATH_MAX_LENGTH]; + + if (playlist->scan_record.content_dir && *playlist->scan_record.content_dir) + { + tmp[0] = '\0'; + path_replace_base_path_and_convert_to_local_file_system(tmp, + playlist->scan_record.content_dir, + playlist->base_content_directory, oldref_len, + playlist->config.base_content_directory, newref_len, + sizeof(tmp)); + playlist_entry_replace_str(&playlist->scan_record.content_dir, tmp); + } + if (playlist->scan_record.dat_file_path && *playlist->scan_record.dat_file_path) + { + tmp[0] = '\0'; + path_replace_base_path_and_convert_to_local_file_system(tmp, + playlist->scan_record.dat_file_path, + playlist->base_content_directory, oldref_len, + playlist->config.base_content_directory, newref_len, + sizeof(tmp)); + playlist_entry_replace_str(&playlist->scan_record.dat_file_path, tmp); + } +} + static void playlist_parse_enter_autofix(playlist_parse_t *p) { playlist_t *playlist = p->playlist; @@ -3265,6 +3359,23 @@ static void playlist_parse_enter_autofix(playlist_parse_t *p) return; } + /* The header alone: its scan record moves as autofix would move + * it, in memory - there are no entries to fix and nothing is + * saved. */ + if (p->header_only) + { + if ( playlist->config.autofix_paths + && !string_is_equal(playlist->base_content_directory, + playlist->config.base_content_directory) + && playlist->base_content_directory + && *playlist->base_content_directory) + playlist_rebase_scan_record(playlist, + strlen(playlist->base_content_directory), + strlen(playlist->config.base_content_directory)); + p->phase = PLAYLIST_PARSE_PHASE_DONE; + return; + } + if ( playlist->config.autofix_paths && !string_is_equal(playlist->base_content_directory, playlist->config.base_content_directory) @@ -3369,6 +3480,11 @@ static int playlist_parse_step_json(playlist_parse_t *p, stop = t; goto stopped; } + if (p->context.flags & JSON_CTX_FLG_HEADER_DONE) + { + stop = RJSON_DONE; + goto stopped; + } } stopped: @@ -3678,35 +3794,8 @@ static int playlist_parse_step_autofix(playlist_parse_t *p, { p->autofix_scan_done = true; - /* Fix scan record content directory */ - if (playlist->scan_record.content_dir && *playlist->scan_record.content_dir) - { - tmp_entry_path[0] = '\0'; - path_replace_base_path_and_convert_to_local_file_system( - tmp_entry_path, - playlist->scan_record.content_dir, - playlist->base_content_directory, p->oldref_len, - playlist->config.base_content_directory, p->newref_len, - sizeof(tmp_entry_path)); - - playlist_entry_replace_str(&playlist->scan_record.content_dir, - tmp_entry_path); - } - - /* Fix scan record arcade DAT file */ - if (playlist->scan_record.dat_file_path && *playlist->scan_record.dat_file_path) - { - tmp_entry_path[0] = '\0'; - path_replace_base_path_and_convert_to_local_file_system( - tmp_entry_path, - playlist->scan_record.dat_file_path, - playlist->base_content_directory, p->oldref_len, - playlist->config.base_content_directory, p->newref_len, - sizeof(tmp_entry_path)); - - playlist_entry_replace_str(&playlist->scan_record.dat_file_path, - tmp_entry_path); - } + /* Fix the scan record's content directory and DAT file */ + playlist_rebase_scan_record(playlist, p->oldref_len, p->newref_len); } } @@ -3729,7 +3818,8 @@ static int playlist_parse_step_autofix(playlist_parse_t *p, return 1; } -playlist_parse_t *playlist_parse_begin(const playlist_config_t *config) +static playlist_parse_t *playlist_parse_begin_ex( + const playlist_config_t *config, bool header_only) { playlist_parse_t *p = NULL; playlist_t *playlist = (playlist_t*)malloc(sizeof(*playlist)); @@ -3770,6 +3860,11 @@ playlist_parse_t *playlist_parse_begin(const playlist_config_t *config) p->playlist = playlist; p->res = true; p->phase = PLAYLIST_PARSE_PHASE_DONE; /* until proven otherwise */ + if ((p->header_only = header_only)) + { + playlist->flags |= CNT_PLAYLIST_FLG_HEADER_ONLY; + p->context.flags |= JSON_CTX_FLG_HEADER_ONLY; + } #if defined(HAVE_COMPRESSION) /* Always use RZIP interface when reading playlists @@ -3860,6 +3955,13 @@ playlist_parse_t *playlist_parse_begin(const playlist_config_t *config) return p; } + /* The old format has no scan settings: a header read is done. */ + if (p->header_only) + { + playlist_parse_enter_autofix(p); + return p; + } + /* Heap-held: at six entries this is 3 KiB even at the console * path length, on the playlist load path that runs from task * threads. On allocation failure the legacy-format file simply @@ -3879,6 +3981,16 @@ playlist_parse_t *playlist_parse_begin(const playlist_config_t *config) return NULL; } +playlist_parse_t *playlist_parse_begin(const playlist_config_t *config) +{ + return playlist_parse_begin_ex(config, false); +} + +playlist_parse_t *playlist_parse_begin_header(const playlist_config_t *config) +{ + return playlist_parse_begin_ex(config, true); +} + int playlist_parse_step(playlist_parse_t *p, bool (*budget_cb)(void *), void *budget_ud) { @@ -3950,6 +4062,8 @@ void playlist_parse_abort(playlist_parse_t *p) void playlist_free_cached(void) { + /* what was written since no longer has a cache to tell */ + playlist_cached_apply_writes(); if (playlist_cached && !(playlist_cached->flags & CNT_PLAYLIST_FLG_CACHED_EXT)) playlist_free(playlist_cached); playlist_cached = NULL; @@ -3980,6 +4094,7 @@ playlist_t *playlist_get_cached(void) * size is the one signal the VFS layer offers portably. */ static bool playlist_cached_is_reusable(const playlist_config_t *config) { + playlist_cached_apply_writes(); if (!playlist_cached || playlist_cached_stale) return false; if (!string_is_equal(playlist_cached->config.path, config->path)) diff --git a/playlist.h b/playlist.h index 7ccc98d26192..0e21c6377b4b 100644 --- a/playlist.h +++ b/playlist.h @@ -259,6 +259,12 @@ playlist_t *playlist_init(const playlist_config_t *config); typedef struct playlist_parse playlist_parse_t; playlist_parse_t *playlist_parse_begin(const playlist_config_t *config); +/* The same, for what comes ahead of the entries only - the defaults + * and the scan record, which the writer puts before them - as a + * playlist with no entries that playlist_write_file() never writes. + * The scan record's paths move to the configured base content + * directory as a full read's autofix would move them, in memory. */ +playlist_parse_t *playlist_parse_begin_header(const playlist_config_t *config); int playlist_parse_step(playlist_parse_t *p, bool (*budget_cb)(void *), void *budget_ud); playlist_t *playlist_parse_end(playlist_parse_t *p); diff --git a/retroarch.c b/retroarch.c index d0606c069d4d..502bbd4ee649 100644 --- a/retroarch.c +++ b/retroarch.c @@ -231,6 +231,7 @@ #include "core.h" #include "configuration.h" #include "list_special.h" +#include "manual_content_scan.h" #ifdef HAVE_CHEATS #include "cheat_manager.h" #endif @@ -390,6 +391,19 @@ enum RA_OPT_MAX_FRAMES_SCREENSHOT_PATH, RA_OPT_SET_SHADER, RA_OPT_DATABASE_SCAN, + RA_OPT_SCAN_REFRESH, + RA_OPT_SCAN_REFRESH_ALL, + RA_OPT_SCAN_PLAYLIST, + RA_OPT_SCAN_DATABASE, + RA_OPT_SCAN_MATCH, + RA_OPT_SCAN_DAT, + RA_OPT_SCAN_CORE, + RA_OPT_SCAN_EXTS, + RA_OPT_SCAN_NO_RECURSIVE, + RA_OPT_SCAN_NO_ARCHIVES, + RA_OPT_SCAN_OVERWRITE, + RA_OPT_SCAN_VALIDATE, + RA_OPT_SCAN_OMIT_DB_REF, RA_OPT_ACCESSIBILITY, RA_OPT_LOAD_MENU_ON_ERROR, RA_OPT_CLOUDSYNC, @@ -7927,9 +7941,59 @@ static void retroarch_print_help(const char *arg0) " how a particular subsystem wants content to be loaded.\n"); #ifdef HAVE_LIBRETRODB + fputs(buf, stdout); + buf[0] = '\0'; + _len = 0; + strlcpy_append(buf, sizeof(buf), &_len, " --scan=PATH|FILE " - "Import content from path.\n"); + "Import content from path. Repeatable.\n" + " " + " The options from --scan-playlist on apply\n" + " " + " to the --scan, --scan-refresh or\n" + " " + " --scan-refresh-all before them. Without\n" + " " + " any, the scan is Fully Automatic.\n"); + strlcpy_append(buf, sizeof(buf), &_len, + " --scan-refresh=NAME|PATH " + "Refresh a playlist by its saved scan settings.\n" + " --scan-refresh-all " + "Refresh each playlist that has saved settings.\n"); + strlcpy_append(buf, sizeof(buf), &_len, + " --scan-playlist=NAME " + "Playlist to add to.\n" + " --scan-database=NAME|first " + "Database to match: one by name, or the first\n" + " " + " that matches.\n" + " --scan-match=MODE " + "Database check: strict, loose or none.\n" + " --scan-dat=FILE " + "Arcade DAT file to match instead, strict unless\n" + " " + " --scan-match=loose.\n"); + strlcpy_append(buf, sizeof(buf), &_len, + " --scan-core=CORE " + "Core: a path, a file name or a display name.\n" + " --scan-exts=EXTS " + "File extensions, separated by commas or spaces.\n" + " --scan-no-recursive " + "Do not scan subfolders.\n" + " --scan-no-archives " + "Do not look inside archives.\n"); + strlcpy_append(buf, sizeof(buf), &_len, + " --scan-overwrite " + "Replace the playlist instead of adding to it.\n" + " --scan-validate " + "Drop entries for missing or unsupported files.\n" + " --scan-omit-db-ref " + "Look up thumbnails by the playlist name.\n"); + + fputs(buf, stdout); + buf[0] = '\0'; + _len = 0; #endif #ifdef HAVE_CLOUDSYNC strlcpy_append(buf, sizeof(buf), &_len, @@ -8260,6 +8324,253 @@ void handle_dbscan_finished(retro_task_t *task, void *task_data, void *user_data, const char *err); #endif +#ifdef HAVE_LIBRETRODB +/* Appends a @kind item for @arg; false if out of memory. */ +static bool retroarch_cli_scan_add(task_dbscan_item_t **items, + size_t *count, size_t *cap, enum task_dbscan_item_kind kind, + const char *arg) +{ + task_dbscan_item_t *item; + + if (*count == *cap) + { + size_t n = *cap ? *cap * 2 : 4; + task_dbscan_item_t *grown = (task_dbscan_item_t*)realloc(*items, + n * sizeof(*grown)); + + if (!grown) + return false; + *items = grown; + *cap = n; + } + item = &(*items)[*count]; + item->kind = kind; + item->overrides = NULL; + item->arg = NULL; + if (arg && !(item->arg = strdup(arg))) + return false; + (*count)++; + return true; +} + +/* Sets scan option @c, one of @opts, on the latest of the @count + * items; a bad one ends the run before the config loads. */ +static void retroarch_cli_scan_option(task_dbscan_item_t *items, + size_t count, const struct option *opts, int c, const char *value) +{ + manual_content_scan_overrides_t *ov = NULL; + const char *err = NULL; + char **field = NULL; + char *copy = NULL; + size_t max_len = PATH_MAX_LENGTH - 1; + + while (opts->val != c) + opts++; + + if (!count) + err = "must follow a --scan"; + else + { + if (!(ov = items[count - 1].overrides)) + ov = items[count - 1].overrides = + manual_content_scan_overrides_new(); + if (opts->has_arg) + copy = strdup(value); + if (!ov || (opts->has_arg && !copy)) + { + fprintf(stderr, "out of memory\n"); + exit(1); + } + + switch (c) + { + case RA_OPT_SCAN_PLAYLIST: + /* A refresh writes its own playlist. */ + if (items[count - 1].kind == TASK_DBSCAN_ITEM_REFRESH) + err = "cannot follow --scan-refresh"; + else if (items[count - 1].kind == TASK_DBSCAN_ITEM_REFRESH_ALL) + err = "cannot follow --scan-refresh-all"; + else + { + field = &ov->playlist; + /* .lpl.tmp fits a file name */ + max_len = NAME_MAX_LENGTH - 9; + } + break; + case RA_OPT_SCAN_DATABASE: + if (string_is_equal(copy, "first")) + { + free(ov->database); + ov->database = NULL; + ov->flags |= MANUAL_CONTENT_SCAN_OVR_FIRST_DB; + } + else + { + ov->flags &= ~MANUAL_CONTENT_SCAN_OVR_FIRST_DB; + field = &ov->database; + max_len = NAME_MAX_LENGTH - 1; + } + break; + case RA_OPT_SCAN_MATCH: + if (!*copy) + err = "needs a value"; + else if (string_is_equal(copy, "strict")) + ov->match = MANUAL_CONTENT_SCAN_USE_DB_STRICT; + else if (string_is_equal(copy, "loose")) + ov->match = MANUAL_CONTENT_SCAN_USE_DB_LOOSE; + else if (string_is_equal(copy, "none")) + ov->match = MANUAL_CONTENT_SCAN_USE_DB_NONE; + else + err = "must be strict, loose or none"; + ov->flags |= MANUAL_CONTENT_SCAN_OVR_MATCH; + break; + case RA_OPT_SCAN_DAT: + field = &ov->dat; + break; + case RA_OPT_SCAN_CORE: + field = &ov->core; + break; + case RA_OPT_SCAN_EXTS: + { + /* Commas become spaces, runs of spaces one, ends trimmed */ + char *out = copy; + const char *in; + + for (in = copy; *in; in++) + { + char ch = (*in == ',') ? ' ' : *in; + if (ch != ' ' || (out > copy && out[-1] != ' ')) + *out++ = ch; + } + if (out > copy && out[-1] == ' ') + out--; + *out = '\0'; + field = &ov->exts; + } + break; + case RA_OPT_SCAN_NO_RECURSIVE: + ov->flags |= MANUAL_CONTENT_SCAN_OVR_NO_RECURSIVE; + break; + case RA_OPT_SCAN_NO_ARCHIVES: + ov->flags |= MANUAL_CONTENT_SCAN_OVR_NO_ARCHIVES; + break; + case RA_OPT_SCAN_OVERWRITE: + ov->flags |= MANUAL_CONTENT_SCAN_OVR_OVERWRITE; + break; + case RA_OPT_SCAN_VALIDATE: + ov->flags |= MANUAL_CONTENT_SCAN_OVR_VALIDATE; + break; + case RA_OPT_SCAN_OMIT_DB_REF: + ov->flags |= MANUAL_CONTENT_SCAN_OVR_OMIT_DB_REF; + break; + } + + if (field) + { + if (!*copy) + err = "needs a value"; + else if (strlen(copy) > max_len) + err = "is too long"; + else + { + free(*field); + *field = copy; + copy = NULL; + } + } + } + + free(copy); + if (err) + { + /* Before any --scan, no console yet on Windows. */ + frontend_driver_attach_console(); + fprintf(stderr, "--%s %s\n", opts->name, err); + exit(1); + } +} + +/* What the chain told a run without --menu. */ +typedef struct +{ + task_dbscan_chain_result_t totals; + bool done; +} retroarch_cli_scan_state_t; + +static void retroarch_cli_scan_done( + const task_dbscan_chain_result_t *res, void *user_data) +{ + retroarch_cli_scan_state_t *st = (retroarch_cli_scan_state_t*)user_data; + + st->totals = *res; + st->done = true; +} + +static bool retroarch_cli_scan_running(void *data) +{ + return !((retroarch_cli_scan_state_t*)data)->done; +} + +static void retroarch_cli_scan_menu_item(const char *name, + enum task_dbscan_outcome outcome, + const task_dbscan_result_t *res, void *user_data) +{ +#ifdef HAVE_MENU + if (res) + handle_dbscan_finished(NULL, NULL, NULL, NULL); +#endif +} + +/* Runs the --scan @items through the scan chain, which owns them. + * With --menu the chain is pushed and the menu goes on. Without, it + * is the whole run: the process exits 1 if any scan failed, else 0 if + * any added an entry, else 2. */ +static void retroarch_cli_scan(task_dbscan_item_t *items, size_t count, + bool menu, const char *content) +{ + retroarch_cli_scan_state_t st; + settings_t *settings = config_get_ptr(); + int reinit_flags = DRIVERS_CMD_ALL & + ~(DRIVER_VIDEO_MASK | DRIVER_AUDIO_MASK | DRIVER_MICROPHONE_MASK | DRIVER_INPUT_MASK | DRIVER_MIDI_MASK); + + drivers_init(settings, reinit_flags, (enum driver_lifetime_flags)0, false); + retroarch_init_task_queue(); + + if (menu) + { + if (!task_push_dbscan_chain(items, count, + retroarch_cli_scan_menu_item, NULL, NULL)) + fprintf(stderr, "out of memory\n"); + return; + } + +#ifdef HAVE_THREADS + /* Threaded for this run whatever the setting: only there does the + * scan get its readers. */ + if (!task_queue_is_threaded()) + { + task_queue_set_threaded(); + task_queue_check(); + } +#endif + + if (content) + fprintf(stderr, "without --menu: ignoring \"%s\"\n", + content); + + memset(&st, 0, sizeof(st)); + if (task_push_dbscan_chain(items, count, NULL, + retroarch_cli_scan_done, &st)) + task_queue_wait(retroarch_cli_scan_running, &st); + else + fprintf(stderr, "out of memory\n"); + + driver_uninit(DRIVERS_CMD_ALL, (enum driver_lifetime_flags)0); + task_queue_deinit(); + exit((!st.done || st.totals.failed) ? 1 : st.totals.added ? 0 : 2); +} +#endif + #ifdef HAVE_CLOUDSYNC /* The sync's verdict, where the waiting caller reads it. */ static void retroarch_cloud_sync_finished(retro_task_t *task, @@ -8313,8 +8624,11 @@ static bool retroarch_parse_input_and_config( access_state_t *access_st = access_state_get_ptr(); #endif #ifdef HAVE_LIBRETRODB - retro_task_callback_t cb_task_dbscan - = NULL; + task_dbscan_item_t *scan_items = NULL; + size_t scan_count = 0; + size_t scan_cap = 0; + const char *scan_conflict = NULL; + bool cli_scan = false; #endif const struct option opts[] = { @@ -8377,6 +8691,19 @@ static bool retroarch_parse_input_and_config( { "entryslot", 1, NULL, 'e' }, #ifdef HAVE_LIBRETRODB { "scan", 1, NULL, RA_OPT_DATABASE_SCAN }, + { "scan-refresh", 1, NULL, RA_OPT_SCAN_REFRESH }, + { "scan-refresh-all", 0, NULL, RA_OPT_SCAN_REFRESH_ALL }, + { "scan-playlist", 1, NULL, RA_OPT_SCAN_PLAYLIST }, + { "scan-database", 1, NULL, RA_OPT_SCAN_DATABASE }, + { "scan-match", 1, NULL, RA_OPT_SCAN_MATCH }, + { "scan-dat", 1, NULL, RA_OPT_SCAN_DAT }, + { "scan-core", 1, NULL, RA_OPT_SCAN_CORE }, + { "scan-exts", 1, NULL, RA_OPT_SCAN_EXTS }, + { "scan-no-recursive", 0, NULL, RA_OPT_SCAN_NO_RECURSIVE }, + { "scan-no-archives", 0, NULL, RA_OPT_SCAN_NO_ARCHIVES }, + { "scan-overwrite", 0, NULL, RA_OPT_SCAN_OVERWRITE }, + { "scan-validate", 0, NULL, RA_OPT_SCAN_VALIDATE }, + { "scan-omit-db-ref", 0, NULL, RA_OPT_SCAN_OMIT_DB_REF }, #endif #ifdef HAVE_CLOUDSYNC { "cloudsync", 0, NULL, RA_OPT_CLOUDSYNC }, @@ -8571,13 +8898,79 @@ static bool retroarch_parse_input_and_config( explicit_menu = true; break; case RA_OPT_DATABASE_SCAN: + case RA_OPT_SCAN_REFRESH: + case RA_OPT_SCAN_REFRESH_ALL: #ifdef HAVE_LIBRETRODB - verbosity_enable(); - retroarch_override_setting_set(RARCH_OVERRIDE_SETTING_DATABASE_SCAN, NULL); + { + enum task_dbscan_item_kind kind = TASK_DBSCAN_ITEM_PATH; + const char *arg = optarg; + + if (c == RA_OPT_SCAN_REFRESH) + { + if (!*arg) + { + frontend_driver_attach_console(); + fprintf(stderr, "--scan-refresh needs a value\n"); + exit(1); + } + kind = TASK_DBSCAN_ITEM_REFRESH; + } + else if (c == RA_OPT_SCAN_REFRESH_ALL) + { + kind = TASK_DBSCAN_ITEM_REFRESH_ALL; + arg = NULL; + } + /* Cut to fit the chain's path buffers, a path could + * name a folder above the one asked for. */ + if (arg && strlen(arg) >= PATH_MAX_LENGTH) + { + frontend_driver_attach_console(); + fprintf(stderr, "--%s is too long\n", + c == RA_OPT_SCAN_REFRESH ? "scan-refresh" : "scan"); + exit(1); + } + cli_scan = true; + /* The totals and errors are the command's output: the + * config's log_verbosity must not turn them off, which + * on Windows detaches the console. */ + verbosity_enable(); + retroarch_override_setting_set(RARCH_OVERRIDE_SETTING_VERBOSITY, NULL); + retroarch_override_setting_set(RARCH_OVERRIDE_SETTING_DATABASE_SCAN, NULL); + if (!retroarch_cli_scan_add(&scan_items, &scan_count, + &scan_cap, kind, arg)) + { + fprintf(stderr, "out of memory\n"); + exit(1); + } + } +#endif + break; + case RA_OPT_SCAN_PLAYLIST: + case RA_OPT_SCAN_DATABASE: + case RA_OPT_SCAN_MATCH: + case RA_OPT_SCAN_DAT: + case RA_OPT_SCAN_CORE: + case RA_OPT_SCAN_EXTS: + case RA_OPT_SCAN_NO_RECURSIVE: + case RA_OPT_SCAN_NO_ARCHIVES: + case RA_OPT_SCAN_OVERWRITE: + case RA_OPT_SCAN_VALIDATE: + case RA_OPT_SCAN_OMIT_DB_REF: +#ifdef HAVE_LIBRETRODB + retroarch_cli_scan_option(scan_items, scan_count, opts, c, + optarg); #endif break; +#if defined(HAVE_LIBRETRODB) && defined(HAVE_NETWORKING) && defined(HAVE_NETWORK_CMD) && defined(HAVE_COMMAND) + case RA_OPT_COMMAND: + scan_conflict = "--command"; + break; +#endif case RA_OPT_CLOUDSYNC: #ifdef HAVE_CLOUDSYNC +#ifdef HAVE_LIBRETRODB + scan_conflict = "--cloudsync"; +#endif /* The sync's summary is the command's output. */ verbosity_enable(); retroarch_override_setting_set(RARCH_OVERRIDE_SETTING_VERBOSITY, NULL); @@ -8604,6 +8997,21 @@ static bool retroarch_parse_input_and_config( } } } +#ifdef HAVE_LIBRETRODB + /* Each of them is the whole run. */ + if (cli_scan && scan_conflict) + { + const char *first = "--scan"; + + if (scan_items[0].kind == TASK_DBSCAN_ITEM_REFRESH) + first = "--scan-refresh"; + else if (scan_items[0].kind == TASK_DBSCAN_ITEM_REFRESH_ALL) + first = "--scan-refresh-all"; + fprintf(stderr, "%s cannot be combined with %s\n", first, + scan_conflict); + exit(1); + } +#endif verbosity_enabled = verbosity_is_enabled(); /* Enable logging to file if verbosity and log-file arguments were passed. * RARCH_OVERRIDE_SETTING_LOG_TO_FILE is set by the RA_OPT_LOG_FILE case above @@ -9025,42 +9433,20 @@ static bool retroarch_parse_input_and_config( } break; case RA_OPT_DATABASE_SCAN: -#ifdef HAVE_LIBRETRODB - { - settings_t *settings = config_get_ptr(); - int reinit_flags = DRIVERS_CMD_ALL & - ~(DRIVER_VIDEO_MASK | DRIVER_AUDIO_MASK | DRIVER_MICROPHONE_MASK | DRIVER_INPUT_MASK | DRIVER_MIDI_MASK); - - drivers_init(settings, reinit_flags, (enum driver_lifetime_flags)0, false); - retroarch_init_task_queue(); -#ifdef HAVE_THREADS - /* Threaded for this run whatever the setting: only - * there does the scan get its readers. */ - if (!explicit_menu && !task_queue_is_threaded()) - { - task_queue_set_threaded(); - task_queue_check(); - } -#endif - -#ifdef HAVE_MENU - if (explicit_menu) - cb_task_dbscan = handle_dbscan_finished; -#endif - - task_push_dbscan(optarg, cb_task_dbscan); - - if (!explicit_menu) - { - /* Command-line scan with no menu: the scan is - * the whole run, so run the queue dry and exit. - * Nothing is drawn meanwhile. */ - task_queue_wait(NULL, NULL); - driver_uninit(DRIVERS_CMD_ALL, (enum driver_lifetime_flags)0); - exit(0); - } - } -#endif + case RA_OPT_SCAN_REFRESH: + case RA_OPT_SCAN_REFRESH_ALL: + case RA_OPT_SCAN_PLAYLIST: + case RA_OPT_SCAN_DATABASE: + case RA_OPT_SCAN_MATCH: + case RA_OPT_SCAN_DAT: + case RA_OPT_SCAN_CORE: + case RA_OPT_SCAN_EXTS: + case RA_OPT_SCAN_NO_RECURSIVE: + case RA_OPT_SCAN_NO_ARCHIVES: + case RA_OPT_SCAN_OVERWRITE: + case RA_OPT_SCAN_VALIDATE: + case RA_OPT_SCAN_OMIT_DB_REF: + /* Taken in the first pass. */ break; case RA_OPT_CLOUDSYNC: #ifdef HAVE_CLOUDSYNC @@ -9105,6 +9491,12 @@ static bool retroarch_parse_input_and_config( } } +#ifdef HAVE_LIBRETRODB + if (scan_items) + retroarch_cli_scan(scan_items, scan_count, explicit_menu, + (optind < argc) ? argv[optind] : NULL); +#endif + #ifdef HAVE_GIT_VERSION RARCH_LOG("RetroArch %s (Git %s)\n", PACKAGE_VERSION, retroarch_git_version); diff --git a/samples/playlist/playlist_parity_test.c b/samples/playlist/playlist_parity_test.c index 156f3b57dae7..08c180a27e03 100644 --- a/samples/playlist/playlist_parity_test.c +++ b/samples/playlist/playlist_parity_test.c @@ -57,6 +57,7 @@ #include "../../core_info.h" #include +#include bool core_info_find(const char *core_path, core_info_t **core_info) { @@ -812,6 +813,209 @@ static void check_cache_reuse(const char *path, bool compressed, playlist_free_cached(); } + +/* A task on a worker writes the playlist file the menu has cached. + * The writer leaves a note for the main thread rather than touching + * the cached playlist, which the main thread frees and installs at + * will; the main thread applies the note when it next asks the cache. + * The worker's write keeps the file's size, so the note alone must + * keep the cache from serving what is on disk no longer. A stress + * round follows: the worker rewrites while the main thread reuses and + * frees the cache (under ASan, and TSan with SANITIZER=thread). */ +struct cache_writer +{ + const char *path; + unsigned rounds; +}; + +static void cache_writer_name(const char *path, const char *name) +{ + playlist_config_t config; + playlist_t *pl; + config_defaults(&config, path); + if (!(pl = playlist_init(&config))) + return; + playlist_set_default_core_name(pl, name); + playlist_write_file(pl); + playlist_free(pl); +} + +static void cache_writer_run(void *arg) +{ + struct cache_writer *w = (struct cache_writer*)arg; + unsigned i; + for (i = 0; i < w->rounds; i++) + cache_writer_name(w->path, (i & 1) ? "Core X" : "Core Y"); +} + +static void lane_cache_worker_write(void) +{ + char path[512]; + playlist_config_t config; + struct cache_writer w; + sthread_t *t; + playlist_t *pl; + unsigned had = failures; + unsigned i; + + snprintf(path, sizeof(path), "%s/worker.lpl", fixture_dir); + CHECK(write_whole(path, json_full), "worker lane: fixture write"); + /* the writer's own form, so every later write has its size */ + cache_writer_name(path, "Core Z"); + config_defaults(&config, path); + + playlist_free_cached(); + CHECK(playlist_init_cached(&config), "worker lane: cached init"); + pl = playlist_get_cached(); + CHECK(pl && streq(playlist_get_default_core_name(pl), "Core Z"), + "worker lane: cached playlist"); + + w.path = path; + w.rounds = 1; + CHECK((t = sthread_create(cache_writer_run, &w)) != NULL, + "worker lane: writer thread"); + if (t) + sthread_join(t); + + CHECK(playlist_init_cached(&config), "worker lane: cached init after"); + pl = playlist_get_cached(); + CHECK(pl && streq(playlist_get_default_core_name(pl), "Core Y"), + "worker lane: a same-size write from a worker was served " + "from the stale cache"); + + /* concurrent: the worker writes while the main thread reuses, + * reads and frees */ + w.rounds = 200; + if ((t = sthread_create(cache_writer_run, &w))) + { + for (i = 0; i < 400; i++) + { + /* Windows replaces the file with MoveFileExW, and an open + * that meets it fails: that read is an empty playlist. */ + if (playlist_init_cached(&config) && (pl = playlist_get_cached())) + CHECK( playlist_get_default_core_name(pl) != NULL +#ifdef _WIN32 + || !playlist_size(pl) +#endif + , "worker lane: stress read"); + if (i & 1) + playlist_free_cached(); + } + sthread_join(t); + } + CHECK(playlist_init_cached(&config), "worker lane: final init"); + pl = playlist_get_cached(); + /* the last of 200 writes, round 199, wrote "Core X" */ + CHECK(pl && streq(playlist_get_default_core_name(pl), "Core X"), + "worker lane: the cache missed the worker's last write"); + playlist_free_cached(); + if (failures == had) + fprintf(stderr, "[pass] worker-write cache lane\n"); +} + +/* The whole file, for comparing it before and after; caller frees. */ +static char *slurp(const char *path, long *len) +{ + char *buf = NULL; + FILE *f = fopen(path, "rb"); + *len = -1; + if (!f) + return NULL; + if ( !fseek(f, 0, SEEK_END) + && (*len = ftell(f)) >= 0 + && !fseek(f, 0, SEEK_SET) + && (buf = (char*)malloc((size_t)*len + 1))) + { + if (fread(buf, 1, (size_t)*len, f) != (size_t)*len) + { + free(buf); + buf = NULL; + } + } + fclose(f); + return buf; +} + +static playlist_t *header_read(const playlist_config_t *config) +{ + playlist_parse_t *p = playlist_parse_begin_header(config); + if (!p) + return NULL; + if (playlist_parse_step(p, NULL, NULL) < 0) + { + playlist_parse_abort(p); + return NULL; + } + return playlist_parse_end(p); +} + +/* playlist_parse_begin_header(): what comes ahead of the entries and + * nothing after; under a moved base content directory the scan record + * moves in memory while the file stays as it was; the result is never + * written; the old format, which has no scan settings, reads empty. */ +static void lane_header_only(void) +{ + char path[512]; + playlist_config_t config; + playlist_t *pl; + char *before, *after; + long blen, alen; + unsigned had = failures; + + snprintf(path, sizeof(path), "%s/header.lpl", fixture_dir); + CHECK(write_whole(path, json_full), "header lane: fixture write"); + config_defaults(&config, path); + + pl = header_read(&config); + CHECK(pl != NULL, "header lane: read"); + if (pl) + { + CHECK(playlist_size(pl) == 0, "header lane: %u entries read", + (unsigned)playlist_size(pl)); + CHECK(streq(playlist_get_default_core_name(pl), "Core A"), + "header lane: default core name"); + CHECK(streq(playlist_get_scan_content_dir(pl), "/games/snes"), + "header lane: scan content dir"); + playlist_free(pl); + } + + /* a base moved since the save: autofix's move, in memory only */ + before = slurp(path, &blen); + playlist_config_set_base_content_directory(&config, "/moved"); + pl = header_read(&config); + /* the suffix in the local form, as autofix converts it */ + CHECK(pl && streq(playlist_get_scan_content_dir(pl), +#ifdef _WIN32 + "/moved\\snes" +#else + "/moved/snes" +#endif + ), "header lane: scan content dir under the moved base"); + if (pl) + { + /* and never written, even when asked */ + playlist_set_default_core_name(pl, "Core Changed"); + playlist_write_file(pl); + playlist_free(pl); + } + after = slurp(path, &alen); + CHECK(before && after && blen == alen && !memcmp(before, after, (size_t)blen), + "header lane: the file changed under a header read"); + free(before); + free(after); + + CHECK(write_whole(path, old_format), "header lane: old format write"); + config_defaults(&config, path); + pl = header_read(&config); + CHECK(pl && playlist_size(pl) == 0 + && !playlist_get_scan_content_dir(pl), + "header lane: the old format reads empty"); + playlist_free(pl); + + if (failures == had) + fprintf(stderr, "[pass] header-only lane\n"); +} + static void lane_cache_reuse(void) { char path[512]; @@ -1660,6 +1864,8 @@ int main(int argc, char *argv[]) lane_budgeted_autofix(); lane_abort_midway(); lane_cache_reuse(); + lane_cache_worker_write(); + lane_header_only(); lane_switch_playlist_never_shows_previous(); lane_switch_supersedes_pending(); lane_saf_slow_reads(); diff --git a/samples/tasks/database/Makefile b/samples/tasks/database/Makefile index 747951601fb1..6d0799fbeff0 100644 --- a/samples/tasks/database/Makefile +++ b/samples/tasks/database/Makefile @@ -249,8 +249,11 @@ $(INDEX_TEST)$(EXE_EXT): $(INDEX_TEST_OBJECTS) # count and refuse reader pushes, count queue worker starts, count an # archive's stored-CRC and directory reads, fail a chosen serial's # query, order the database and content lists, count reader passes, -# hold chosen lookups, count an archive's member hits a step and hold -# a chosen index build in one step. +# hold chosen lookups, count an archive's member hits a step, hold +# a chosen index build in one step, tell which thread the refresh +# walks and playlist loads ran on, fail a chosen task_init, fail the +# playlist folder walk, and count the file system calls the main +# thread makes. SCAN_TEST_WRAPS := -Wl,--wrap=libretrodb_open -Wl,--wrap=task_queue_park \ -Wl,--wrap=intfstream_crc_step -Wl,--wrap=rm3u_is_m3u \ -Wl,--wrap=task_nbio_slice_open -Wl,--wrap=task_nbio_slice_close \ @@ -264,7 +267,12 @@ SCAN_TEST_WRAPS := -Wl,--wrap=libretrodb_open -Wl,--wrap=task_queue_park \ -Wl,--wrap=database_info_dir_init_from_list \ -Wl,--wrap=database_info_index_job_step \ -Wl,--wrap=task_nbio_slice_within_budget \ - -Wl,--wrap=fill_pathname_join_delim + -Wl,--wrap=fill_pathname_join_delim \ + -Wl,--wrap=dir_list_iter_new -Wl,--wrap=dir_list_iter_step \ + -Wl,--wrap=playlist_parse_begin_header -Wl,--wrap=task_init \ + -Wl,--wrap=path_stat -Wl,--wrap=path_is_valid \ + -Wl,--wrap=path_is_directory -Wl,--wrap=path_get_size \ + -Wl,--wrap=path_resolve_realpath ifeq ($(HAVE_THREADS), 1) SCAN_TEST_WRAPS += -Wl,--wrap=sthread_create endif diff --git a/samples/tasks/database/database_scan_test.c b/samples/tasks/database/database_scan_test.c index 85a3d864c219..4d31da853798 100644 --- a/samples/tasks/database/database_scan_test.c +++ b/samples/tasks/database/database_scan_test.c @@ -35,6 +35,19 @@ #include #include #include +#ifdef _WIN32 +#include +#define cap_dup _dup +#define cap_dup2 _dup2 +#define cap_close _close +#define cap_fileno _fileno +#else +#include +#define cap_dup dup +#define cap_dup2 dup2 +#define cap_close close +#define cap_fileno fileno +#endif #include #include @@ -58,6 +71,7 @@ #include "../../../list_special.h" #include "../../../configuration.h" #include "../../../verbosity.h" +#include "../../../retroarch_types.h" #include "../../../libretro-db/libretrodb.h" #ifdef HAVE_SMBCLIENT #include "../../../libretro-common/vfs/vfs_implementation_smb.h" @@ -125,12 +139,40 @@ static void end_step_hold(void) retro_atomic_store_release_int(&end_readers_out, n ? 0 : 1); } +/* The English rows, for the messages a check compares whole. */ +static const struct +{ + enum msg_hash_enums id; + const char *str; +} msg_hash_us_rows[] = { +#undef MSG_HASH +#define MSG_HASH(Id, str) { Id, str }, +#include "../../../intl/msg_hash_us.h" +#undef MSG_HASH +}; + +static const char *msg_hash_us_row(enum msg_hash_enums msg) +{ + size_t i; + + for (i = 0; i < sizeof(msg_hash_us_rows) / sizeof(msg_hash_us_rows[0]); + i++) + if (msg_hash_us_rows[i].id == msg) + return msg_hash_us_rows[i].str; + return ""; +} + const char *msg_hash_to_str_us(enum msg_hash_enums msg) { if (msg == MSG_SCANNING_OF_DIRECTORY_FINISHED && end_hold) end_step_hold(); + if ( msg == MSG_PLAYLIST_MANAGER_REFRESH_ALL_DONE + || msg == MSG_MANUAL_CONTENT_SCAN_ALREADY_RUNNING) + return msg_hash_us_row(msg); if (msg == MSG_MANUAL_CONTENT_SCAN_INVALID_CONTENT) return "invalid content"; + if (msg == MSG_SCANNING_NO_DATABASE) + return "no database"; return ""; } @@ -159,6 +201,8 @@ settings_t *config_get_ptr(void) } static bool invalid_content_pushed = false; +/* How often a refused scan or chain said why. */ +static unsigned busy_pushed = 0; /* Additional stubs for retroarch-core symbols referenced transitively, * to avoid pulling in retroarch.c, runloop.c, frontend drivers, and the @@ -172,12 +216,18 @@ void runloop_msg_queue_push(const char *msg, size_t len, (void)flush; (void)title; (void)icon; (void)category; if (msg && !strcmp(msg, "invalid content")) invalid_content_pushed = true; + if (msg && !strcmp(msg, + msg_hash_to_str(MSG_MANUAL_CONTENT_SCAN_ALREADY_RUNNING))) + busy_pushed++; } +/* Whether the scan prints what --scan prints. */ +static bool cli_output; + bool retroarch_override_setting_is_set(unsigned enum_idx, void *data) { - (void)enum_idx; (void)data; - return false; + (void)data; + return cli_output && enum_idx == RARCH_OVERRIDE_SETTING_DATABASE_SCAN; } void ui_companion_driver_notify_refresh(void) @@ -484,7 +534,8 @@ bool __wrap_task_queue_is_threaded(void) return __real_task_queue_is_threaded(); } -/* A stream of slow_size bytes sleeps slow_ms before its first quantum; +/* A stream of slow_size bytes sleeps slow_ms before its first quantum, + * then waits while slow_hold is set, at most HOLD_USEC (below); * slow_in counts those sleeps begun, slow_back those ended (slow_begun * holds reader_begun as each of the first four ends), slow_by_scan * those in the scan's tick and slow_on_scan those on the thread that @@ -493,7 +544,8 @@ bool __wrap_task_queue_is_threaded(void) * counted by who makes them, a reader's on the main thread apart, and * those of begun_size bytes as each begins (reader_begun those a * reader begins); so are reads begun once cb_fired is set. A reader's - * read of a stream of fail_size bytes fails, counted in reader_fails. + * read of a stream of fail_size bytes fails, counted in reader_fails; + * every read of a stream of err_size bytes fails, the scan's own too. * The scan's first quantum of a stream of hold_size bytes waits, at * most HOLD_USEC, until hold_want reads of begun_size bytes have begun, * a slow one if slow_size is set and gate_n readers are in the gate @@ -503,6 +555,7 @@ bool __wrap_task_queue_is_threaded(void) static uint32_t slow_size; static unsigned slow_ms; static retro_atomic_int_t slow_in; +static retro_atomic_int_t slow_hold; static retro_atomic_int_t slow_back; static int slow_begun[4]; static retro_atomic_int_t slow_on_scan; @@ -518,6 +571,7 @@ static retro_atomic_int_t cb_fired; static retro_atomic_int_t reads_after_cb; static uint32_t fail_size; static retro_atomic_int_t reader_fails; +static uint32_t err_size; static uint32_t hold_size; static int hold_want; static int gate_n; @@ -562,6 +616,7 @@ int64_t __wrap_intfstream_crc_step(intfstream_internal_t *intf, if (slow_size && size == (int64_t)slow_size && first) { int k; + retro_time_t until = cpu_features_get_time_usec() + HOLD_USEC; if (retro_atomic_load_acquire_size(&scan_thread_tid) == (size_t)current_tid()) @@ -570,6 +625,9 @@ int64_t __wrap_intfstream_crc_step(intfstream_internal_t *intf, retro_atomic_fetch_add_int(&slow_by_scan, 1); retro_atomic_fetch_add_int(&slow_in, 1); retro_sleep(slow_ms); + while ( retro_atomic_load_acquire_int(&slow_hold) + && cpu_features_get_time_usec() < until) + retro_sleep(1); if ((k = retro_atomic_fetch_add_int(&slow_back, 1)) < 4) slow_begun[k] = retro_atomic_load_acquire_int(&reader_begun); } @@ -604,6 +662,8 @@ int64_t __wrap_intfstream_crc_step(intfstream_internal_t *intf, #endif } } + if (err_size && size == (int64_t)err_size) + return -1; if (fail_size && size == (int64_t)fail_size && !scan) { retro_atomic_fetch_add_int(&reader_fails, 1); @@ -965,13 +1025,36 @@ static retro_time_t push_cancel_usec; static retro_atomic_int_t pushes_tried; static retro_atomic_int_t pushes_ok; +/* The chain's untitled steps: the prep_cancel_at'th pushed (from 1) is + * cancelled once its playlist-folder walk or playlist load begins. + * Set only between runs. */ +static unsigned prep_cancel_at; +static unsigned prep_pushes; +static retro_task_t *prep_cancel_task; + +static void prep_cancel_begun(void) +{ + retro_task_t *task = prep_cancel_task; + + if (task) + { + prep_cancel_task = NULL; + task_set_flags(task, RETRO_TASK_FLG_CANCELLED, true); + } +} + bool __real_task_queue_push(retro_task_t *task); bool __wrap_task_queue_push(retro_task_t *task) { int n; if (!(task_get_flags(task) & RETRO_TASK_FLG_PARALLEL)) + { + if ( prep_cancel_at && !task->title + && ++prep_pushes == prep_cancel_at) + prep_cancel_task = task; return __real_task_queue_push(task); + } n = retro_atomic_fetch_add_int(&pushes_tried, 1) + 1; if (push_fail_from && (unsigned)n >= push_fail_from) return false; @@ -1889,13 +1972,20 @@ static int file_contains(const char *path, const char *needle) static bool loop_active = true; static bool scan_completed = false; static retro_time_t cb_usec; +/* What the last run_scan() was handed: its user data is this. */ +static task_dbscan_result_t scan_result; +static bool scan_err; static void scan_cb(retro_task_t *task, void *task_data, void *user_data, const char *err) { - (void)task; (void)task_data; (void)user_data; + (void)task; if (err && *err) fprintf(stderr, " scan reported an error: %s\n", err); + scan_err = (err != NULL); + if (user_data && task_data) + *(task_dbscan_result_t*)user_data = + *(const task_dbscan_result_t*)task_data; retro_atomic_store_release_int(&cb_fired, 1); cb_usec = cpu_features_get_time_usec(); scan_completed = true; @@ -1927,7 +2017,14 @@ static bool run_scan(const char *dir) retro_atomic_store_release_int(&rd_watch_wholes, 0); retro_atomic_store_release_int(&look_hits, 0); retro_atomic_store_release_int(&look_late, 0); - if (!task_push_dbscan(dir, scan_cb)) + /* Kept unless the callback gets both pointers. */ + scan_result.added = 9999; + scan_result.duplicates = 9999; + scan_result.unmatched = 9999; + scan_result.errors = 9999; + scan_result.failed = true; + scan_err = false; + if (!task_push_dbscan(dir, scan_cb, &scan_result)) return false; started = time(NULL); @@ -1954,7 +2051,7 @@ static unsigned run_scan_counted(const char *dir) loop_active = true; scan_completed = false; - if (!task_push_dbscan(dir, scan_cb)) + if (!task_push_dbscan(dir, scan_cb, NULL)) return 0; started = time(NULL); @@ -2306,6 +2403,243 @@ static void single_lane(const char *root, const char *pl_dir, free(kept); } +/* Checks the result the last run_scan() was handed. */ +static void check_result(const char *what, unsigned added, + unsigned duplicates, unsigned unmatched, unsigned errors, bool failed) +{ + char detail[128]; + + sprintf(detail, "%u added, %u duplicates, %u unmatched, %u errors%s", + scan_result.added, scan_result.duplicates, scan_result.unmatched, + scan_result.errors, scan_result.failed ? ", failed" : ""); + check( scan_completed + && scan_result.added == added + && scan_result.duplicates == duplicates + && scan_result.unmatched == unmatched + && scan_result.errors == errors + && scan_result.failed == failed, + what, scan_completed ? detail : "refused or timed out"); +} + +/* Deletes the playlists an earlier run left in @dir. */ +static void delete_playlists(const char *dir) +{ + size_t i; + struct string_list *l = dir_list_new(dir, "lpl", false, false, false, + false); + + if (!l) + return; + for (i = 0; i < l->size; i++) + filestream_delete(l->elems[i].data); + string_list_free(l); +} + +/* A scan counts each file by what became of it, across every playlist + * it writes: an archive is not unmatched for the "??" it prints before + * it is expanded, and a loose scan adds what matches nothing. */ +static void counts_lane(const char *root, const char *pl_dir, + uint32_t crc_a, uint32_t sz_a, uint32_t crc_d, uint32_t sz_d) +{ + char *dir = (char*)malloc(512); + char *dat_dir = (char*)malloc(512); + char *m3u_dir = (char*)malloc(512); + char *res_pl = (char*)malloc(512); + char *p = (char*)malloc(1024); + uint8_t *d = (uint8_t*)malloc(sz_a > sz_d ? sz_a : sz_d); + uint32_t i; + FILE *f; + + if (!dir || !dat_dir || !m3u_dir || !res_pl || !p || !d) + { + check(0, "counts lane", "out of memory"); + goto end; + } + + sprintf(dir, "%s/scan_result", root); + sprintf(dat_dir, "%s/scan_result_dat", root); + sprintf(m3u_dir, "%s/scan_result_m3u", root); + sprintf(res_pl, "%s/scan_result_pl", root); + path_mkdir(dir); + path_mkdir(dat_dir); + path_mkdir(m3u_dir); + path_mkdir(res_pl); + + sprintf(p, "%s/01_alpha.bin", dir); + if (!write_content(p, crc_a, sz_a)) + goto fixture; + sprintf(p, "%s/02_none.bin", dir); + if (!write_content(p, 0x0BADF00Du, 1024)) + goto fixture; + /* Matched by the CRC its member stores. */ + for (i = 0; i < sz_d - 4; i++) + d[i] = (uint8_t)((i * 53 + 7) & 0xFF); + crc_force(d, sz_d - 4, crc_d, d + sz_d - 4); + sprintf(p, "%s/03_bundle.zip", dir); + if (!write_zip(p, "inner.bin", d, sz_d, crc_d)) + goto fixture; + /* Its member stores no CRC: expanded, and the member matches + * nothing. */ + for (i = 0; i < sz_a - 4; i++) + d[i] = (uint8_t)((i * 31 + 5) & 0xFF); + crc_force(d, sz_a - 4, crc_a, d + sz_a - 4); + sprintf(p, "%s/04_expand.zip", dir); + if (!write_zip(p, "inner.bin", d, sz_a, 0)) + goto fixture; + + sprintf(p, "%s/01_alpha.bin", dat_dir); + if (!write_content(p, crc_a, sz_a)) + goto fixture; + sprintf(p, "%s/02_none.bin", dat_dir); + if (!write_content(p, 0x0BADF00Du, 1024)) + goto fixture; + sprintf(p, "%s/Game (Disc 1).bin", m3u_dir); + if (!write_content(p, crc_a, sz_a)) + goto fixture; + sprintf(p, "%s/Game (Disc 2).bin", m3u_dir); + if (!write_content(p, crc_a, sz_a)) + goto fixture; + sprintf(p, "%s/Game.m3u", m3u_dir); + if (!(f = fopen(p, "w"))) + goto fixture; + fputs("Game (Disc 1).bin\nGame (Disc 2).bin\n", f); + fclose(f); + + sprintf(p, "%s/scan_result.dat", root); + if (!(f = fopen(p, "w"))) + goto fixture; + fputs("\n\n" + "
Result
\n" + "Alpha\n" + "\n", f); + fclose(f); + + delete_playlists(res_pl); + strlcpy(config_get_ptr()->paths.directory_playlist, res_pl, + sizeof(config_get_ptr()->paths.directory_playlist)); + + run_scan(dir); + check_result("a scan counts what it adds and what matches nothing", + 2, 0, 2, 0, false); + run_scan(dir); + check_result("a rescan counts what it already has", 0, 2, 2, 0, false); + + delete_playlists(res_pl); + manual_content_scan_set_menu_scan_method( + MANUAL_CONTENT_SCAN_METHOD_CUSTOM); + manual_content_scan_set_menu_scan_use_db( + MANUAL_CONTENT_SCAN_USE_DB_LOOSE); + run_scan(dir); + check_result("a loose scan adds what matches nothing", 4, 0, 0, 0, false); + + delete_playlists(res_pl); + manual_content_scan_set_menu_scan_use_db( + MANUAL_CONTENT_SCAN_USE_DB_DAT_STRICT); + strlcpy(manual_content_scan_get_dat_file_path_ptr(), p, + manual_content_scan_get_dat_file_path_size()); + run_scan(dat_dir); + check_result("a DAT scan counts what the DAT leaves out", 1, 0, 1, 0, false); + + manual_content_scan_get_dat_file_path_ptr()[0] = '\0'; + manual_content_scan_set_menu_scan_use_db( + MANUAL_CONTENT_SCAN_USE_DB_STRICT); + manual_content_scan_set_menu_scan_method( + MANUAL_CONTENT_SCAN_METHOD_AUTOMATIC); + + /* An m3u counts once, as the one entry its discs fold into. */ + delete_playlists(res_pl); + run_scan(m3u_dir); + check_result("an m3u is added, not also unmatched", 1, 0, 0, 0, false); + run_scan(m3u_dir); + check_result("an m3u rescan is one duplicate", 0, 1, 0, 0, false); + + /* One playlist, so the loose scan's own entry for the m3u and the + * one its discs fold into meet there. */ + delete_playlists(res_pl); + manual_content_scan_set_menu_scan_method( + MANUAL_CONTENT_SCAN_METHOD_CUSTOM); + manual_content_scan_set_menu_scan_use_db( + MANUAL_CONTENT_SCAN_USE_DB_LOOSE); + manual_content_scan_set_menu_system_name( + MANUAL_CONTENT_SCAN_SYSTEM_NAME_CONTENT_DIR, NULL); + run_scan(m3u_dir); + check_result("a loose m3u is added, not also a duplicate", + 1, 0, 0, 0, false); + run_scan(m3u_dir); + check_result("a loose m3u rescan is one duplicate", 0, 1, 0, 0, false); + manual_content_scan_set_menu_system_name( + MANUAL_CONTENT_SCAN_SYSTEM_NAME_AUTO, NULL); + manual_content_scan_set_menu_scan_use_db( + MANUAL_CONTENT_SCAN_USE_DB_STRICT); + manual_content_scan_set_menu_scan_method( + MANUAL_CONTENT_SCAN_METHOD_AUTOMATIC); + + strlcpy(config_get_ptr()->paths.directory_playlist, pl_dir, + sizeof(config_get_ptr()->paths.directory_playlist)); + goto end; + +fixture: + check(0, "fixture", "could not write the counts lane"); +end: + free(dir); + free(dat_dir); + free(m3u_dir); + free(res_pl); + free(p); + free(d); +} + +#define ERR_SIZE 1536 + +/* A file the scan cannot read is counted as an error; the scan goes + * on and does not fail. */ +static void errors_lane(const char *root, const char *pl_dir, + uint32_t crc_a, uint32_t sz_a) +{ + static const char m3u[] = "Missing (Disc 1).bin\n"; + char dir[512], m3u_dir[512], res_pl[512], p[1024]; + FILE *f; + + sprintf(dir, "%s/scan_errors", root); + sprintf(m3u_dir, "%s/scan_errors_m3u", root); + sprintf(res_pl, "%s/scan_errors_pl", root); + path_mkdir(dir); + path_mkdir(m3u_dir); + path_mkdir(res_pl); + sprintf(p, "%s/01_alpha.bin", dir); + if (!write_content(p, crc_a, sz_a)) + goto fixture; + sprintf(p, "%s/02_unread.bin", dir); + if (!write_content(p, 0x0E770001u, ERR_SIZE)) + goto fixture; + + delete_playlists(res_pl); + strlcpy(config_get_ptr()->paths.directory_playlist, res_pl, + sizeof(config_get_ptr()->paths.directory_playlist)); + err_size = ERR_SIZE; + run_scan(dir); + err_size = 0; + check_result("a file the scan cannot read is an error", 1, 0, 0, 1, + false); + + sprintf(p, "%s/Game.m3u", m3u_dir); + if (!(f = fopen(p, "wb"))) + goto fixture; + fputs(m3u, f); + fclose(f); + err_size = sizeof(m3u) - 1; + run_scan(m3u_dir); + err_size = 0; + check_result("an m3u the scan cannot read is an error, not also " + "unmatched", 0, 0, 0, 1, false); + strlcpy(config_get_ptr()->paths.directory_playlist, pl_dir, + sizeof(config_get_ptr()->paths.directory_playlist)); + return; + +fixture: + check(0, "fixture", "could not write the errors lane"); +} + /* The queue the next scans run on: threaded where threads are built, * else the regular one, so every build runs the same checks. */ static void use_queue(bool threaded) @@ -2344,7 +2678,7 @@ static retro_time_t run_scan_cancelled(const char *dir, bool on_stuck) loop_active = true; scan_completed = false; - if (!task_push_dbscan(dir, scan_cb)) + if (!task_push_dbscan(dir, scan_cb, NULL)) return -1; finder.func = find_scan; finder.userdata = NULL; @@ -2446,6 +2780,251 @@ static void use_db(const char *db_dir) sizeof(config_get_ptr()->paths.path_content_database)); } +/* A scan with no database to look in, or a DAT it cannot load, says + * it failed. */ +static void failed_lane(const char *root, const char *pl_dir, + const char *db_dir, uint32_t crc_a, uint32_t sz_a) +{ + char *dir = (char*)malloc(512); + char *nodb = (char*)malloc(512); + char *res_pl = (char*)malloc(512); + char *p = (char*)malloc(1024); + FILE *f; + bool refused; + + if (!dir || !nodb || !res_pl || !p) + { + check(0, "failed lane", "out of memory"); + goto end; + } + + sprintf(dir, "%s/scan_failed", root); + sprintf(nodb, "%s/scan_failed_nodb", root); + sprintf(res_pl, "%s/scan_failed_pl", root); + path_mkdir(dir); + path_mkdir(res_pl); + + sprintf(p, "%s/01_alpha.bin", dir); + if (!write_content(p, crc_a, sz_a)) + goto fixture; + sprintf(p, "%s/scan_failed.dat", root); + if (!(f = fopen(p, "w"))) + goto fixture; + fputs("\n" + "\n", f); + fclose(f); + + strlcpy(config_get_ptr()->paths.directory_playlist, res_pl, + sizeof(config_get_ptr()->paths.directory_playlist)); + + use_db(nodb); + run_scan(dir); + check_result("a missing database folder fails the scan", 0, 0, 1, 0, true); + check(scan_err, "and reports it as an empty folder is", + scan_err ? "error" : "no error"); + use_db(""); + run_scan(dir); + check_result("as does no database folder set", 0, 0, 1, 0, true); + use_db(db_dir); + check(scan_err, "and reports it too", + scan_err ? "error" : "no error"); + + /* Not a rescan of the folder the last scan set. */ + refused = !run_scan("/"); + check(refused, "a folder the scan settings reject is refused", + refused ? "refused" : "scanned"); + + manual_content_scan_set_menu_scan_method( + MANUAL_CONTENT_SCAN_METHOD_CUSTOM); + manual_content_scan_set_menu_scan_use_db( + MANUAL_CONTENT_SCAN_USE_DB_DAT_STRICT); + strlcpy(manual_content_scan_get_dat_file_path_ptr(), p, + manual_content_scan_get_dat_file_path_size()); + run_scan(dir); + check_result("a DAT that does not load fails the scan", 0, 0, 0, 0, true); + + manual_content_scan_get_dat_file_path_ptr()[0] = '\0'; + manual_content_scan_set_menu_scan_use_db( + MANUAL_CONTENT_SCAN_USE_DB_STRICT); + manual_content_scan_set_menu_scan_method( + MANUAL_CONTENT_SCAN_METHOD_AUTOMATIC); + strlcpy(config_get_ptr()->paths.directory_playlist, pl_dir, + sizeof(config_get_ptr()->paths.directory_playlist)); + goto end; + +fixture: + check(0, "fixture", "could not write the failed lane"); +end: + free(dir); + free(nodb); + free(res_pl); + free(p); +} + +/* stdout and stderr while capture_begin() has them in files. */ +static int cap_out = -1; +static int cap_err = -1; + +static void capture_end(void) +{ + fflush(stdout); + fflush(stderr); + if (cap_out >= 0) + { + cap_dup2(cap_out, cap_fileno(stdout)); + cap_close(cap_out); + cap_out = -1; + } + if (cap_err >= 0) + { + cap_dup2(cap_err, cap_fileno(stderr)); + cap_close(cap_err); + cap_err = -1; + } +} + +/* Sends stdout to @out and stderr to @err until capture_end(). The + * scan prints to the streams themselves, so their descriptors move. */ +static bool capture_begin(const char *out, const char *err) +{ + FILE *o = fopen(out, "w"); + FILE *e = fopen(err, "w"); + bool ok = false; + + fflush(stdout); + fflush(stderr); + if (o && e) + { + cap_out = cap_dup(cap_fileno(stdout)); + cap_err = cap_dup(cap_fileno(stderr)); + ok = cap_out >= 0 && cap_err >= 0 + && cap_dup2(cap_fileno(o), cap_fileno(stdout)) >= 0 + && cap_dup2(cap_fileno(e), cap_fileno(stderr)) >= 0; + } + if (o) + fclose(o); + if (e) + fclose(e); + if (!ok) + capture_end(); + return ok; +} + +/* Scans @dir with stdout in @out and stderr in @err; false if the + * streams could not be moved or the scan did not finish. */ +static bool capture_scan(const char *dir, const char *out, const char *err, + bool cli) +{ + bool captured; + + cli_output = cli; + if ((captured = capture_begin(out, err))) + { + run_scan(dir); + capture_end(); + } + cli_output = false; + check(captured && scan_completed, "--scan output to files", + !captured ? out : scan_completed ? "" : "timed out"); + return captured && scan_completed; +} + +/* What --scan prints, to files: the marks plain, the totals line on + * stdout, and on stderr a file the scan could not read and a scan + * that failed, which prints no totals. Without --scan, nothing. + * Rescans errors_lane()'s folder. */ +static void output_lane(const char *root, const char *pl_dir, + const char *db_dir) +{ + char *dir = (char*)malloc(512); + char *nodb = (char*)malloc(512); + char *empty = (char*)malloc(512); + char *res_pl = (char*)malloc(512); + char *out = (char*)malloc(512); + char *err = (char*)malloc(512); + char *log = (char*)malloc(512); + char *want = (char*)malloc(1024); + bool logged; + + if (!dir || !nodb || !empty || !res_pl || !out || !err || !log || !want) + { + check(0, "output lane", "out of memory"); + goto end; + } + + sprintf(dir, "%s/scan_errors", root); + sprintf(nodb, "%s/scan_output_nodb", root); + sprintf(empty, "%s/scan_output_empty", root); + sprintf(res_pl, "%s/scan_errors_pl", root); + sprintf(out, "%s/scan_output.out", root); + sprintf(err, "%s/scan_output.err", root); + sprintf(log, "%s/scan_output.log", root); + path_mkdir(empty); + strlcpy(config_get_ptr()->paths.directory_playlist, res_pl, + sizeof(config_get_ptr()->paths.directory_playlist)); + + err_size = ERR_SIZE; + if (capture_scan(dir, out, err, true)) + { + /* No "\n" in the needles: Windows ends these lines "\r\n". */ + sprintf(want, "%s: 0 added, 1 duplicates, 0 unmatched, 1 errors", + dir); + check(file_contains(out, want), "--scan prints its totals on stdout", + want); + sprintf(want, " !! \"%s/02_unread.bin\"", dir); + check(file_contains(err, want), + "and a file it could not read on stderr", want); + check( file_contains(out, " == \"") + && !file_contains(out, "\x1B[") && !file_contains(err, "\x1B["), + "with no colour codes in a file", out); + sprintf(want, "\"%s\"...", dir); + check(file_contains(out, want), + "and starts with the folder it scans", want); + } + + retro_main_log_file_init(log, false); + verbosity_enable(); + logged = capture_scan(dir, out, err, false); + verbosity_disable(); + retro_main_log_file_deinit(); + if (logged) + { + check(path_get_size(out) == 0 && path_get_size(err) == 0, + "a scan without --scan prints nothing", out); + sprintf(want, "\"%s\"...", dir); + check(file_contains(log, want), "but logs the folder it scans", want); + } + err_size = 0; + + use_db(nodb); + if (capture_scan(dir, out, err, true)) + { + sprintf(want, "%s: no database", dir); + check(file_contains(err, want) && !file_contains(out, " added, "), + "a failed scan says so on stderr, no totals", want); + } + use_db(db_dir); + + if (capture_scan(empty, out, err, true)) + { + sprintf(want, "%s: invalid content", empty); + check(file_contains(err, want) && !file_contains(out, " added, "), + "as does a scan that cannot start", want); + } + + strlcpy(config_get_ptr()->paths.directory_playlist, pl_dir, + sizeof(config_get_ptr()->paths.directory_playlist)); +end: + free(dir); + free(nodb); + free(empty); + free(res_pl); + free(out); + free(err); + free(log); + free(want); +} + static bool write_reader_db(const char *db_dir) { char p[1024]; @@ -2653,7 +3232,7 @@ static void reader_quit_lane(const char *root, const char *db_dir) slow_reset(READER_STUCK_SIZE, READER_QUIT_MS); loop_active = true; scan_completed = false; - if (!task_push_dbscan(dir, scan_cb)) + if (!task_push_dbscan(dir, scan_cb, NULL)) { check(0, "quit: scan started", "task_push_dbscan refused"); return; @@ -4588,48 +5167,253 @@ static void loose_held_lane(const char *root, const char *pl_dir, check(0, "fixture", "could not write the loose held set"); } -#ifdef HAVE_RRAR -/* Rar archives take the archive passes: a first member's hit, in a - * database whose core opens rar, names the archive; a later member's, - * in one whose core does not, is held; an archive of several members - * is read whole only while a database whose cores open rar holds a rar - * record of its size. */ -static void rar_lane(const char *root, const char *pl_dir, bool threaded) -{ - static uint8_t d[4][MF_MEMBER_LEN]; - static const uint32_t crc[4] = { - 0x0BF80001u, 0x0BF80011u, 0x0BF80012u, 0x0BF80002u }; - char db[512], noclaim[512], dir[512], p[1024]; - static char cart_lpl[1024], member_lpl[1024]; - char what[64], detail[64]; - db_rec_t cart[3], held, nc; - zip_entry_t e[2]; - uint32_t cart_bytes, multi_bytes; - int cart_reads, multi_reads; - bool started; - unsigned j; - const char *q = threaded ? "(threaded)" : "(regular)"; - - sprintf(db, "%s/rar_db", root); - sprintf(noclaim, "%s/rar_noclaim_db", root); - sprintf(dir, "%s/rar", root); +/* A refresh of a playlist that saved no database name leaves the + * database to the scan, whatever the refresh before it read. */ +static void refresh_db_name_lane(const char *root) +{ + static const char lpl[] = + "{\n" + " \"version\": \"1.5\",\n" + " \"scan_content_dir\": \"%s\",\n" + " \"scan_database_name\": \"%s\",\n" + " \"scan_search_recursively\": true,\n" + " \"scan_search_archives\": true,\n" + " \"scan_db_usage\": 0,\n" + " \"items\": []\n" + "}\n"; + static char text[2048]; + manual_content_scan_task_config_t *cfg = + (manual_content_scan_task_config_t*)calloc(1, sizeof(*cfg)); + playlist_config_t *pl_cfg = + (playlist_config_t*)calloc(1, sizeof(*pl_cfg)); + playlist_t *named = NULL; + playlist_t *plain = NULL; + char db[512], dir[512], pl[512], p[1024]; + db_rec_t rec; + bool ran; + + if (!cfg || !pl_cfg) + { + check(0, "refresh database name lane", "out of memory"); + goto end; + } + sprintf(db, "%s/refresh_name_db", root); + sprintf(dir, "%s/refresh_name", root); + sprintf(pl, "%s/refresh_name_pl", root); path_mkdir(db); - path_mkdir(noclaim); path_mkdir(dir); - for (j = 0; j < 4; j++) - fill_forced(d[j], MF_MEMBER_LEN, 101 + 2 * j, crc[j]); - memset(e, 0, sizeof(e)); - e[0].len = MF_MEMBER_LEN; - e[1].len = MF_MEMBER_LEN; - - e[0].name = "m.rd"; - e[0].data = d[0]; - e[0].crc = crc[0]; - sprintf(p, "%s/rar_cart.rar", dir); - if (!write_rar_entries(p, e, 1)) + path_mkdir(pl); + set_rec(&rec, "Named Cart", 0x4E414D45u, 1024); + sprintf(p, "%s/Test Named.rdb", db); + if (!write_db_set(p, &rec, 1)) + goto fixture; + pl_cfg->capacity = COLLECTION_SIZE; + sprintf(p, "%s/Named.lpl", pl); + sprintf(text, lpl, dir, "Test Named.lpl"); + playlist_config_set_path(pl_cfg, p); + if ( !filestream_write_file(p, text, strlen(text)) + || !(named = playlist_init(pl_cfg))) + goto fixture; + sprintf(p, "%s/Plain.lpl", pl); + sprintf(text, lpl, dir, ""); + playlist_config_set_path(pl_cfg, p); + if ( !filestream_write_file(p, text, strlen(text)) + || !(plain = playlist_init(pl_cfg))) goto fixture; - cart_bytes = rar_bytes(e, 1); - set_rec(&cart[0], "RAR Cart", crc[0], MF_MEMBER_LEN); + + /* Back to back: the second call's frame is where the first left + * its database name. */ + ran = manual_content_scan_set_menu_from_playlist(named, db, false) + == MANUAL_CONTENT_SCAN_PLAYLIST_REFRESH_OK + && manual_content_scan_set_menu_from_playlist(plain, db, false) + == MANUAL_CONTENT_SCAN_PLAYLIST_REFRESH_OK + && manual_content_scan_get_task_config(cfg, pl); + check(ran && !*cfg->database_name, + "refresh, no saved database: the scan picks", + !ran ? "refused" : *cfg->database_name ? cfg->database_name + : "automatic"); + ran = manual_content_scan_set_menu_from_playlist(named, db, false) + == MANUAL_CONTENT_SCAN_PLAYLIST_REFRESH_OK + && manual_content_scan_get_task_config(cfg, pl); + check(ran && !strcmp(cfg->database_name, "Test Named.lpl"), + "refresh, a saved database: that one", + ran ? cfg->database_name : "refused"); + goto restore; + +fixture: + check(0, "fixture", "could not write the refresh playlists"); +restore: + *manual_content_scan_get_validate_entries_ptr() = false; + manual_content_scan_set_menu_scan_db_select( + MANUAL_CONTENT_SCAN_SELECT_DB_AUTO, NULL); + manual_content_scan_get_system_name_custom_ptr()[0] = '\0'; + manual_content_scan_set_menu_system_name( + MANUAL_CONTENT_SCAN_SYSTEM_NAME_AUTO, NULL); + manual_content_scan_set_menu_scan_method( + MANUAL_CONTENT_SCAN_METHOD_AUTOMATIC); +end: + if (named) + playlist_free(named); + if (plain) + playlist_free(plain); + free(pl_cfg); + free(cfg); +} + +static unsigned file_count(const char *path, const char *needle); + +/* Skip Database References names a match after the playlist it is + * written to, and only when the scan has one playlist: left to the + * scan, a match goes to its database's playlist. */ +static void omit_ref_lane(const char *root, const char *pl_dir, + bool threaded) +{ + static char cart_lpl[1024], folder_lpl[1024], custom_lpl[1024]; + char db[512], dir[512], pl[512], p[1024], what[80]; + db_rec_t cart; + bool started; + const char *q = threaded ? "(threaded)" : "(regular)"; + + sprintf(db, "%s/omit_ref_db", root); + sprintf(dir, "%s/omit_ref", root); + sprintf(pl, "%s/omit_ref_pl", root); + path_mkdir(db); + path_mkdir(dir); + path_mkdir(pl); + sprintf(p, "%s/om.ct", dir); + if (!write_content(p, 0x0A1E0F01u, 1024)) + goto fixture; + set_rec(&cart, "Omit Cart", 0x0A1E0F01u, 1024); + sprintf(p, "%s/Test Cart Gate.rdb", db); + if (!write_db_set(p, &cart, 1)) + goto fixture; + + sprintf(cart_lpl, "%s/Test Cart Gate.lpl", pl); + sprintf(folder_lpl, "%s/omit_ref.lpl", pl); + sprintf(custom_lpl, "%s/Omit Custom.lpl", pl); + filestream_delete(cart_lpl); + filestream_delete(folder_lpl); + use_queue(threaded); + use_db(db); + strlcpy(config_get_ptr()->paths.directory_playlist, pl, + sizeof(config_get_ptr()->paths.directory_playlist)); + *manual_content_scan_get_omit_db_ref_ptr() = true; + + manual_content_scan_set_menu_scan_method( + MANUAL_CONTENT_SCAN_METHOD_CUSTOM); + started = run_scan(dir) && scan_completed; + sprintf(what, "omit ref, Automatic target %s", q); + check( started && file_contains(cart_lpl, "om.ct") + && file_contains(cart_lpl, "\"db_name\": \"Test Cart Gate.lpl\"") + && !path_is_valid(folder_lpl), what, "Test Cart Gate.lpl"); + + /* Fully Automatic keeps the value Custom set. */ + filestream_delete(cart_lpl); + filestream_delete(folder_lpl); + manual_content_scan_set_menu_scan_method( + MANUAL_CONTENT_SCAN_METHOD_AUTOMATIC); + started = run_scan(dir) && scan_completed; + sprintf(what, "omit ref, Fully Automatic %s", q); + check( started && file_contains(cart_lpl, "om.ct") + && file_contains(cart_lpl, "\"db_name\": \"Test Cart Gate.lpl\"") + && !path_is_valid(folder_lpl), what, "Test Cart Gate.lpl"); + + manual_content_scan_set_menu_scan_method( + MANUAL_CONTENT_SCAN_METHOD_CUSTOM); + strlcpy(manual_content_scan_get_system_name_custom_ptr(), + "Omit Custom", + manual_content_scan_get_system_name_custom_size()); + manual_content_scan_set_menu_system_name( + MANUAL_CONTENT_SCAN_SYSTEM_NAME_CUSTOM, NULL); + filestream_delete(custom_lpl); + started = run_scan(dir) && scan_completed; + sprintf(what, "omit ref, Custom target: its name %s", q); + check( started + && file_contains(custom_lpl, "\"db_name\": \"Omit Custom.lpl\""), + what, "Omit Custom.lpl"); + + /* A loose scan's non-match too, not the folder's name. */ + sprintf(p, "%s/om_none.ct", dir); + if (!write_content(p, 0x0A1E0F02u, 1024)) + goto fixture; + manual_content_scan_set_menu_scan_use_db( + MANUAL_CONTENT_SCAN_USE_DB_LOOSE); + filestream_delete(custom_lpl); + started = run_scan(dir) && scan_completed; + manual_content_scan_set_menu_scan_use_db( + MANUAL_CONTENT_SCAN_USE_DB_STRICT); + filestream_delete(p); + sprintf(what, "omit ref, Custom target, loose: a non-match %s", q); + check( started && file_contains(custom_lpl, "om_none.ct") + && file_contains(custom_lpl, "om.ct") + && file_count(custom_lpl, "\"db_name\": \"Omit Custom.lpl\"") == 2 + && !file_contains(custom_lpl, "\"db_name\": \"omit_ref.lpl\"") + && !file_contains(custom_lpl, "\"db_name\": \"Test Cart Gate"), + what, custom_lpl); + + *manual_content_scan_get_omit_db_ref_ptr() = false; + filestream_delete(custom_lpl); + started = run_scan(dir) && scan_completed; + sprintf(what, "omit ref off: the database's name %s", q); + check( started + && file_contains(custom_lpl, "\"db_name\": \"Test Cart Gate.lpl\""), + what, "Test Cart Gate.lpl"); + + manual_content_scan_get_system_name_custom_ptr()[0] = '\0'; + manual_content_scan_set_menu_system_name( + MANUAL_CONTENT_SCAN_SYSTEM_NAME_AUTO, NULL); + manual_content_scan_set_menu_scan_method( + MANUAL_CONTENT_SCAN_METHOD_AUTOMATIC); + strlcpy(config_get_ptr()->paths.directory_playlist, pl_dir, + sizeof(config_get_ptr()->paths.directory_playlist)); + return; + +fixture: + check(0, "fixture", "could not write the omit ref set"); +} + +#ifdef HAVE_RRAR +/* Rar archives take the archive passes: a first member's hit, in a + * database whose core opens rar, names the archive; a later member's, + * in one whose core does not, is held; an archive of several members + * is read whole only while a database whose cores open rar holds a rar + * record of its size. */ +static void rar_lane(const char *root, const char *pl_dir, bool threaded) +{ + static uint8_t d[4][MF_MEMBER_LEN]; + static const uint32_t crc[4] = { + 0x0BF80001u, 0x0BF80011u, 0x0BF80012u, 0x0BF80002u }; + char db[512], noclaim[512], dir[512], p[1024]; + static char cart_lpl[1024], member_lpl[1024]; + char what[64], detail[64]; + db_rec_t cart[3], held, nc; + zip_entry_t e[2]; + uint32_t cart_bytes, multi_bytes; + int cart_reads, multi_reads; + bool started; + unsigned j; + const char *q = threaded ? "(threaded)" : "(regular)"; + + sprintf(db, "%s/rar_db", root); + sprintf(noclaim, "%s/rar_noclaim_db", root); + sprintf(dir, "%s/rar", root); + path_mkdir(db); + path_mkdir(noclaim); + path_mkdir(dir); + for (j = 0; j < 4; j++) + fill_forced(d[j], MF_MEMBER_LEN, 101 + 2 * j, crc[j]); + memset(e, 0, sizeof(e)); + e[0].len = MF_MEMBER_LEN; + e[1].len = MF_MEMBER_LEN; + + e[0].name = "m.rd"; + e[0].data = d[0]; + e[0].crc = crc[0]; + sprintf(p, "%s/rar_cart.rar", dir); + if (!write_rar_entries(p, e, 1)) + goto fixture; + cart_bytes = rar_bytes(e, 1); + set_rec(&cart[0], "RAR Cart", crc[0], MF_MEMBER_LEN); e[0].name = "a.rd"; e[0].data = d[1]; @@ -5123,7 +5907,7 @@ static void run_scan_cancel_at(const char *dir, const step_count_t *c, loop_active = true; scan_completed = false; - if (!task_push_dbscan(dir, scan_cb)) + if (!task_push_dbscan(dir, scan_cb, NULL)) return; finder.func = find_scan; finder.userdata = NULL; @@ -7404,7 +8188,7 @@ static retro_time_t run_scan_switching(const char *dir, bool *started, scan_completed = false; *stalls = 0; *switches = 0; - if (!(*started = task_push_dbscan(dir, scan_cb))) + if (!(*started = task_push_dbscan(dir, scan_cb, NULL))) return 0; #ifdef HAVE_THREADS finder.func = find_scan; @@ -7573,7 +8357,7 @@ static void switch_stuck_lane(const char *root, const char *pl_dir, slow_reset(READER_STUCK_SIZE, READER_STUCK_MS); loop_active = true; scan_completed = false; - if (!task_push_dbscan(dir, scan_cb)) + if (!task_push_dbscan(dir, scan_cb, NULL)) { check(0, "a switch past a stuck read: scan completes", "task_push_dbscan refused"); @@ -7672,6 +8456,1852 @@ static void share_gate_lane(const char *root, const char *db_dir) #endif } +/* Cores for core_lookup_lane(): two files with one file id, two cores + * with one display name, a display name with a slash, a file id + * with an underscore and two file names that differ only in case. */ +static bool write_lookup_cores(const char *info_dir, const char *core_dir) +{ + static const char *const info[][3] = { + { "dup", "Dup Test", "dupx" }, + { "same1", "Same Name Test", "samex" }, + { "same2", "Same Name Test", "samex" }, + { "slash", "Slash / Test (Lookup)", "slashx" }, + { "two_part", "Two Part Test", "twox" } + }; + static const char *const so[] = { + "dup_libretro.so", "dup_libretro_android.so", + "same1_libretro.so", "same2_libretro.so", "slash_libretro.so", + "two_part_libretro.so", "twin_libretro.so", "Twin_libretro.so" + }; + char p[1024]; + size_t i; + FILE *f; + + for (i = 0; i < sizeof(info) / sizeof(info[0]); i++) + { + sprintf(p, "%s/%s_libretro.info", info_dir, info[i][0]); + if (!(f = fopen(p, "w"))) + return false; + fprintf(f, "display_name = \"%s\"\nsupported_extensions = \"%s\"\n", + info[i][1], info[i][2]); + fclose(f); + } + for (i = 0; i < sizeof(so) / sizeof(so[0]); i++) + { + sprintf(p, "%s/%s", core_dir, so[i]); + if (!(f = fopen(p, "wb"))) + return false; + fputs("\177ELF", f); + fclose(f); + } + return true; +} + +/* The core list's path for the core whose file is @file. */ +static const char *lookup_core_path(const char *file) +{ + core_info_list_t *list = NULL; + size_t i; + + core_info_get_list(&list); + for (i = 0; list && i < list->count; i++) + { + core_info_t *info = core_info_get(list, i); + if (info && info->path && !strcmp(path_basename(info->path), file)) + return info->path; + } + return ""; +} + +static char *ovr_str(const char *s) +{ + char *d = (char*)malloc(strlen(s) + 1); + if (d) + strcpy(d, s); + return d; +} + +/* A fresh record in place of @ov, which is freed. */ +static manual_content_scan_overrides_t *ovr_fresh( + manual_content_scan_overrides_t *ov) +{ + manual_content_scan_overrides_free(ov); + return manual_content_scan_overrides_new(); +} + +static manual_content_scan_task_config_t *ovr_cfg; +static bool ovr_cfg_ok; +static const char *ovr_dir; +static const char *ovr_pl; + +/* A --scan item's settings: a reset, the folder, @ov. The task config + * lands in ovr_cfg, and ovr_cfg_ok says whether it could be built. */ +static enum manual_content_scan_override_status ovr_run( + const manual_content_scan_overrides_t *ov, + const struct string_list *rdb, unsigned *count) +{ + enum manual_content_scan_override_status st; + + manual_content_scan_reset_settings(); + manual_content_scan_set_menu_content_dir(ovr_dir); + st = manual_content_scan_apply_overrides(ov, rdb, count); + ovr_cfg_ok = st == MANUAL_CONTENT_SCAN_OVR_OK + && manual_content_scan_get_task_config(ovr_cfg, ovr_pl); + return st; +} + +/* After a Custom setup, every setting is back to its default, and + * Custom over a reset builds the task Fully Automatic does. */ +static void reset_lane(const char *root, const char *pl_dir) +{ + manual_content_scan_task_config_t *cfg = + (manual_content_scan_task_config_t*)malloc(sizeof(*cfg)); + char dir[512]; + bool set_up; + + if (!cfg) + { + check(0, "reset lane", "out of memory"); + return; + } + sprintf(dir, "%s/scan_reset", root); + path_mkdir(dir); + + manual_content_scan_set_menu_scan_method( + MANUAL_CONTENT_SCAN_METHOD_CUSTOM); + manual_content_scan_set_menu_content_dir(dir); + manual_content_scan_set_menu_scan_use_db( + MANUAL_CONTENT_SCAN_USE_DB_DAT_LOOSE); + strlcpy(manual_content_scan_get_dat_file_path_ptr(), "/x/reset.dat", + manual_content_scan_get_dat_file_path_size()); + manual_content_scan_set_menu_scan_db_select( + MANUAL_CONTENT_SCAN_SELECT_DB_SPECIFIC, "Test Alpha"); + strlcpy(manual_content_scan_get_system_name_custom_ptr(), "Reset Custom", + manual_content_scan_get_system_name_custom_size()); + manual_content_scan_set_menu_system_name( + MANUAL_CONTENT_SCAN_SYSTEM_NAME_CUSTOM, NULL); + set_up = manual_content_scan_set_menu_core_name( + MANUAL_CONTENT_SCAN_CORE_SET, "Scan Test") + && !strcmp(manual_content_scan_get_file_exts(), "bin cue gdi m3u zip"); + strlcpy(manual_content_scan_get_file_exts_custom_ptr(), "rst", + manual_content_scan_get_file_exts_custom_size()); + *manual_content_scan_get_search_recursively_ptr() = false; + *manual_content_scan_get_search_archives_ptr() = false; + *manual_content_scan_get_omit_db_ref_ptr() = true; + *manual_content_scan_get_filter_dat_content_ptr() = true; + *manual_content_scan_get_overwrite_playlist_ptr() = true; + *manual_content_scan_get_validate_entries_ptr() = true; + check(set_up, "reset: the Custom setup took", "core Scan Test"); + + manual_content_scan_reset_settings(); + check( manual_content_scan_get_scan_method_enum() + == MANUAL_CONTENT_SCAN_METHOD_AUTOMATIC + && manual_content_scan_get_scan_use_db_enum() + == MANUAL_CONTENT_SCAN_USE_DB_STRICT + && manual_content_scan_get_scan_db_select_enum() + == MANUAL_CONTENT_SCAN_SELECT_DB_AUTO + && manual_content_scan_get_menu_system_name_type() + == MANUAL_CONTENT_SCAN_SYSTEM_NAME_AUTO, + "reset: method, database, playlist", "Automatic defaults"); + check( !*manual_content_scan_get_content_dir_ptr() + && !*manual_content_scan_get_dat_file_path_ptr() + && !*manual_content_scan_get_system_name_custom_ptr() + && !*manual_content_scan_get_file_exts_custom_ptr() + && !*manual_content_scan_get_file_exts(), + "reset: every path and name emptied", ""); + check( *manual_content_scan_get_search_recursively_ptr() + && *manual_content_scan_get_search_archives_ptr() + && !*manual_content_scan_get_omit_db_ref_ptr() + && !*manual_content_scan_get_filter_dat_content_ptr() + && !*manual_content_scan_get_overwrite_playlist_ptr() + && !*manual_content_scan_get_validate_entries_ptr(), + "reset: the switches", "recursive, archives on, rest off"); + + manual_content_scan_set_menu_content_dir(dir); + manual_content_scan_set_menu_scan_method( + MANUAL_CONTENT_SCAN_METHOD_CUSTOM); + check( manual_content_scan_get_task_config(cfg, pl_dir) + && !cfg->core_set && !*cfg->core_name && !*cfg->core_path + && !*cfg->file_exts && !cfg->file_exts_custom_set + && !*cfg->system_name && !*cfg->playlist_file + && !*cfg->database_name + && cfg->search_recursively && cfg->search_archives + && !cfg->filter_dat_content && !cfg->overwrite_playlist + && !cfg->validate_entries && !cfg->omit_db_reference + && cfg->db_usage == MANUAL_CONTENT_SCAN_USE_DB_STRICT + && cfg->db_selection == MANUAL_CONTENT_SCAN_SELECT_DB_AUTO, + "reset: Custom builds the automatic task", ""); + + manual_content_scan_reset_settings(); + free(cfg); +} + +/* Each override lands in the task config. */ +static void overrides_lane(const char *root, const char *pl_dir) +{ + manual_content_scan_overrides_t *ov = NULL; + struct string_list *rdb = string_list_new(); + union string_list_elem_attr attr; + char dir[512], dat[1024], txt[1024], empty[1024], missing[1024]; + char p[1024]; + unsigned count = 0; + enum manual_content_scan_override_status st; + FILE *f; + + ovr_cfg = (manual_content_scan_task_config_t*)malloc(sizeof(*ovr_cfg)); + if (!rdb || !ovr_cfg) + { + check(0, "overrides lane", "out of memory"); + goto end; + } + attr.i = 0; + string_list_append(rdb, "/db/Test Alpha.rdb", attr); + string_list_append(rdb, "/db/Test Beta.rdb", attr); + + sprintf(dir, "%s/scan_ovr", root); + sprintf(dat, "%s/scan_ovr/ovr.dat", root); + sprintf(txt, "%s/scan_ovr/ovr.txt", root); + sprintf(empty, "%s/scan_ovr/empty.dat", root); + sprintf(missing, "%s/scan_ovr/missing.dat", root); + path_mkdir(dir); + if (!(f = fopen(dat, "w"))) + goto fixture; + fputs("\n\n", f); + fclose(f); + if (!(f = fopen(txt, "w"))) + goto fixture; + fputs("not a dat\n", f); + fclose(f); + if (!(f = fopen(empty, "w"))) + goto fixture; + fclose(f); + remove(missing); + ovr_dir = dir; + ovr_pl = pl_dir; + + check( ovr_run(NULL, rdb, &count) == MANUAL_CONTENT_SCAN_OVR_OK + && manual_content_scan_get_scan_method_enum() + == MANUAL_CONTENT_SCAN_METHOD_AUTOMATIC, + "overrides: none leave Fully Automatic", ""); + + /* Core: path, name and extensions */ + ov = ovr_fresh(ov); + ov->core = ovr_str("Scan Test"); + strlcpy(p, lookup_core_path("test_libretro.so"), sizeof(p)); + st = ovr_run(ov, rdb, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok + && manual_content_scan_get_scan_method_enum() + == MANUAL_CONTENT_SCAN_METHOD_CUSTOM + && ovr_cfg->core_set && count == 1 + && !strcmp(ovr_cfg->core_name, "Scan Test") + && !strcmp(ovr_cfg->core_path, p) + && !strcmp(ovr_cfg->file_exts, "bin|cue|gdi|m3u|zip") + && !ovr_cfg->file_exts_custom_set, + "overrides: core, its path and extensions", ovr_cfg->core_path); + + /* Custom extensions win over the core's, scrubbed */ + ov->exts = ovr_str(".BIN gam"); + st = ovr_run(ov, rdb, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok + && ovr_cfg->file_exts_custom_set + && !strcmp(ovr_cfg->file_exts, "bin|gam"), + "overrides: extensions", ovr_cfg->file_exts); + + /* Database Check without a DAT */ + ov = ovr_fresh(ov); + ov->flags = MANUAL_CONTENT_SCAN_OVR_MATCH; + ov->match = MANUAL_CONTENT_SCAN_USE_DB_LOOSE; + st = ovr_run(ov, rdb, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok + && ovr_cfg->db_usage == MANUAL_CONTENT_SCAN_USE_DB_LOOSE, + "overrides: match loose", ""); + ov->match = MANUAL_CONTENT_SCAN_USE_DB_STRICT; + st = ovr_run(ov, rdb, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok + && ovr_cfg->db_usage == MANUAL_CONTENT_SCAN_USE_DB_STRICT, + "overrides: match strict", ""); + ov->match = MANUAL_CONTENT_SCAN_USE_DB_NONE; + sprintf(p, "%s/scan_ovr.lpl", pl_dir); + st = ovr_run(ov, rdb, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok + && ovr_cfg->db_usage == MANUAL_CONTENT_SCAN_USE_DB_NONE + && !strcmp(ovr_cfg->system_name, "scan_ovr") + && !strcmp(ovr_cfg->playlist_file, p), + "overrides: match none, named after the folder", + ovr_cfg->system_name); + + /* A DAT: strict unless loose, never with none */ + ov = ovr_fresh(ov); + ov->dat = ovr_str(dat); + st = ovr_run(ov, rdb, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok + && ovr_cfg->db_usage == MANUAL_CONTENT_SCAN_USE_DB_DAT_STRICT + && ovr_cfg->filter_dat_content + && !strcmp(ovr_cfg->dat_file_path, dat), + "overrides: DAT alone is strict", ovr_cfg->dat_file_path); + ov->flags = MANUAL_CONTENT_SCAN_OVR_MATCH; + ov->match = MANUAL_CONTENT_SCAN_USE_DB_LOOSE; + st = ovr_run(ov, rdb, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok + && ovr_cfg->db_usage == MANUAL_CONTENT_SCAN_USE_DB_DAT_LOOSE + && !ovr_cfg->filter_dat_content + && !strcmp(ovr_cfg->dat_file_path, dat), + "overrides: DAT with loose", ""); + ov->match = MANUAL_CONTENT_SCAN_USE_DB_STRICT; + st = ovr_run(ov, rdb, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok + && ovr_cfg->db_usage == MANUAL_CONTENT_SCAN_USE_DB_DAT_STRICT, + "overrides: DAT with strict", ""); + ov->match = MANUAL_CONTENT_SCAN_USE_DB_NONE; + st = ovr_run(ov, rdb, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_DAT_WITHOUT_MATCH + && !*manual_content_scan_get_dat_file_path_ptr(), + "overrides: DAT with none fails untouched", ""); + + /* DAT errors */ + ov = ovr_fresh(ov); + ov->dat = ovr_str(txt); + check(ovr_run(ov, rdb, &count) == MANUAL_CONTENT_SCAN_OVR_NOT_A_DAT, + "overrides: DAT with another extension", "not a DAT"); + ov = ovr_fresh(ov); + ov->dat = ovr_str(missing); + check(ovr_run(ov, rdb, &count) == MANUAL_CONTENT_SCAN_OVR_NOT_A_DAT, + "overrides: DAT missing", "not a DAT"); + ov = ovr_fresh(ov); + ov->dat = ovr_str(empty); + check(ovr_run(ov, rdb, &count) == MANUAL_CONTENT_SCAN_OVR_NOT_A_DAT, + "overrides: DAT empty", "not a DAT"); + + /* A refresh's saved DAT: the match picks its mode, none drops it */ + ov = ovr_fresh(ov); + ov->flags = MANUAL_CONTENT_SCAN_OVR_MATCH; + { + static const enum manual_content_scan_db_usage + match[3] = { MANUAL_CONTENT_SCAN_USE_DB_LOOSE, + MANUAL_CONTENT_SCAN_USE_DB_STRICT, + MANUAL_CONTENT_SCAN_USE_DB_NONE }; + static const enum manual_content_scan_db_usage + want[3] = { MANUAL_CONTENT_SCAN_USE_DB_DAT_LOOSE, + MANUAL_CONTENT_SCAN_USE_DB_DAT_STRICT, + MANUAL_CONTENT_SCAN_USE_DB_NONE }; + static const char *const what[3] = { + "overrides: saved DAT with loose", + "overrides: saved DAT with strict", + "overrides: saved DAT with none drops it" }; + int i; + + for (i = 0; i < 3; i++) + { + manual_content_scan_reset_settings(); + manual_content_scan_set_menu_content_dir(dir); + manual_content_scan_set_menu_scan_use_db( + MANUAL_CONTENT_SCAN_USE_DB_DAT_STRICT); + strlcpy(manual_content_scan_get_dat_file_path_ptr(), dat, + manual_content_scan_get_dat_file_path_size()); + ov->match = match[i]; + st = manual_content_scan_apply_overrides(ov, rdb, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK + && manual_content_scan_get_task_config(ovr_cfg, pl_dir) + && ovr_cfg->db_usage == want[i] + && (i == 2 + ? !*manual_content_scan_get_dat_file_path_ptr() + : !strcmp(ovr_cfg->dat_file_path, dat)), + what[i], ""); + } + } + + /* Database To Match */ + ov = ovr_fresh(ov); + ov->database = ovr_str("Test Alpha"); + sprintf(p, "%s/Test Alpha.lpl", pl_dir); + st = ovr_run(ov, rdb, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok + && ovr_cfg->db_selection == MANUAL_CONTENT_SCAN_SELECT_DB_SPECIFIC + && !strcmp(ovr_cfg->database_name, "Test Alpha.lpl") + && !strcmp(ovr_cfg->system_name, "Test Alpha") + && !strcmp(ovr_cfg->playlist_file, p), + "overrides: database by name", ovr_cfg->database_name); + check(ovr_run(ov, NULL, &count) == MANUAL_CONTENT_SCAN_OVR_NO_SUCH_DATABASE, + "overrides: no database installed", "no such database"); + ov = ovr_fresh(ov); + ov->database = ovr_str("Test Nope"); + check(ovr_run(ov, rdb, &count) == MANUAL_CONTENT_SCAN_OVR_NO_SUCH_DATABASE, + "overrides: an unknown database", "no such database"); + ov = ovr_fresh(ov); + ov->flags = MANUAL_CONTENT_SCAN_OVR_FIRST_DB; + st = ovr_run(ov, rdb, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok + && ovr_cfg->db_selection + == MANUAL_CONTENT_SCAN_SELECT_DB_AUTO_FIRST_MATCH, + "overrides: first matching database", ""); + + /* Target Playlist */ + ov = ovr_fresh(ov); + ov->playlist = ovr_str("Test Beta"); + sprintf(p, "%s/Test Beta.lpl", pl_dir); + st = ovr_run(ov, rdb, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok + && manual_content_scan_get_menu_system_name_type() + == MANUAL_CONTENT_SCAN_SYSTEM_NAME_DATABASE + && !strcmp(ovr_cfg->system_name, "Test Beta") + && !strcmp(ovr_cfg->playlist_file, p), + "overrides: playlist named after a database", ovr_cfg->system_name); + ov->database = ovr_str("Test Alpha"); + st = ovr_run(ov, rdb, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok + && manual_content_scan_get_menu_system_name_type() + == MANUAL_CONTENT_SCAN_SYSTEM_NAME_CUSTOM + && !strcmp(ovr_cfg->system_name, "Test Beta") + && !strcmp(ovr_cfg->playlist_file, p) + && !strcmp(ovr_cfg->database_name, "Test Alpha.lpl"), + "overrides: and another database: Custom", ovr_cfg->database_name); + free(ov->playlist); + ov->playlist = ovr_str("Test Alpha"); + st = ovr_run(ov, rdb, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok + && manual_content_scan_get_menu_system_name_type() + == MANUAL_CONTENT_SCAN_SYSTEM_NAME_DATABASE + && !strcmp(ovr_cfg->system_name, "Test Alpha"), + "overrides: and the same database", ovr_cfg->system_name); + /* As a refresh of a database playlist leaves them */ + ov = ovr_fresh(ov); + ov->database = ovr_str("Test Alpha"); + sprintf(p, "%s/Test Beta.lpl", pl_dir); + manual_content_scan_reset_settings(); + manual_content_scan_set_menu_content_dir(ovr_dir); + manual_content_scan_set_menu_system_name( + MANUAL_CONTENT_SCAN_SYSTEM_NAME_DATABASE, "Test Beta"); + manual_content_scan_set_menu_scan_db_select( + MANUAL_CONTENT_SCAN_SELECT_DB_SPECIFIC, "Test Beta"); + st = manual_content_scan_apply_overrides(ov, rdb, &count); + ovr_cfg_ok = st == MANUAL_CONTENT_SCAN_OVR_OK + && manual_content_scan_get_task_config(ovr_cfg, pl_dir); + check( ovr_cfg_ok + && !strcmp(ovr_cfg->playlist_file, p) + && !strcmp(ovr_cfg->database_name, "Test Alpha.lpl"), + "overrides: a refresh's database playlist keeps its name", + ovr_cfg_ok ? ovr_cfg->playlist_file : "no task config"); + ov = ovr_fresh(ov); + ov->playlist = ovr_str("My: List?"); + sprintf(p, "%s/My_ List_.lpl", pl_dir); + st = ovr_run(ov, rdb, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok + && manual_content_scan_get_menu_system_name_type() + == MANUAL_CONTENT_SCAN_SYSTEM_NAME_CUSTOM + && !strcmp(ovr_cfg->system_name, "My_ List_") + && !strcmp(ovr_cfg->playlist_file, p), + "overrides: custom playlist, scrubbed", ovr_cfg->system_name); + + /* The switches */ + ov = ovr_fresh(ov); + ov->flags = MANUAL_CONTENT_SCAN_OVR_NO_RECURSIVE + | MANUAL_CONTENT_SCAN_OVR_NO_ARCHIVES + | MANUAL_CONTENT_SCAN_OVR_OVERWRITE + | MANUAL_CONTENT_SCAN_OVR_VALIDATE + | MANUAL_CONTENT_SCAN_OVR_OMIT_DB_REF; + st = ovr_run(ov, rdb, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok + && !ovr_cfg->search_recursively && !ovr_cfg->search_archives + && ovr_cfg->overwrite_playlist && ovr_cfg->validate_entries + && ovr_cfg->omit_db_reference, + "overrides: recursive, archives, overwrite, validate, omit", ""); + + /* A step that fails stops the ones after it */ + ov = ovr_fresh(ov); + ov->core = ovr_str("No Such Core"); + ov->database = ovr_str("Test Alpha"); + st = ovr_run(ov, rdb, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_NO_SUCH_CORE && count == 0 + && manual_content_scan_get_scan_db_select_enum() + == MANUAL_CONTENT_SCAN_SELECT_DB_AUTO + && manual_content_scan_get_scan_method_enum() + == MANUAL_CONTENT_SCAN_METHOD_AUTOMATIC, + "overrides: an error stops the rest", ""); + goto end; + +fixture: + check(0, "fixture", "could not write the DAT files"); +end: + manual_content_scan_overrides_free(ov); + manual_content_scan_reset_settings(); + string_list_free(rdb); + free(ovr_cfg); + ovr_cfg = NULL; +} + +/* --scan-core: a path, then a file name, then a display name; more + * than one match in the step that decides is ambiguous. */ +static void core_lookup_lane(const char *root) +{ + manual_content_scan_overrides_t *ov = NULL; + char elsewhere[512], p[1024]; + char dup_path[1024]; + unsigned count; + size_t i; + enum manual_content_scan_override_status st; + const char *core_name = NULL; + static const char *const by_name[] = { + "gamma", "gamma_libretro", "gamma_libretro.so", "gamma_libretro.dll", + "gamma_libretro_android.so", "Gamma Test" }; + static const char *const two_part[] = { + "two_part", "two_part_libretro_ios.dylib" }; + static const char *const other_case[] = { + "GAMMA", "Gamma_libretro.so", "Gamma_Libretro.dll", + "GAMMA_LIBRETRO_ANDROID.SO" }; + /* What each name finds; "" for both */ + static const char *const twin[][2] = { + { "twin", "twin_libretro.so" }, + { "Twin_libretro.so", "Twin_libretro.so" }, + { "TWIN", "" } }; + bool twins; + FILE *f; + + ovr_cfg = (manual_content_scan_task_config_t*)malloc(sizeof(*ovr_cfg)); + if (!ovr_cfg) + { + check(0, "core lookup lane", "out of memory"); + return; + } + sprintf(elsewhere, "%s/scan_lookup_elsewhere", root); + path_mkdir(elsewhere); + sprintf(p, "%s/gamma_libretro.so", elsewhere); + if ((f = fopen(p, "wb"))) + { + fputs("\177ELF", f); + fclose(f); + } + ovr_dir = elsewhere; + ovr_pl = elsewhere; + + ov = ovr_fresh(ov); + ov->core = ovr_str(lookup_core_path("gamma_libretro.so")); + count = 9; + st = ovr_run(ov, NULL, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok && count == 1 + && !strcmp(ovr_cfg->core_name, "Gamma Test") + && !strcmp(ovr_cfg->core_path, ov->core), + "core lookup: by path", ovr_cfg->core_path); + + free(ov->core); + ov->core = ovr_str(p); + count = 9; + check( ovr_run(ov, NULL, &count) == MANUAL_CONTENT_SCAN_OVR_NO_SUCH_CORE + && count == 0, + "core lookup: a path outside the core folder", "no such core"); + + for (i = 0; i < sizeof(by_name) / sizeof(by_name[0]); i++) + { + free(ov->core); + ov->core = ovr_str(by_name[i]); + st = ovr_run(ov, NULL, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok && count == 1 + && !strcmp(ovr_cfg->core_name, "Gamma Test"), + "core lookup: by file or display name", by_name[i]); + } + + for (i = 0; i < sizeof(two_part) / sizeof(two_part[0]); i++) + { + free(ov->core); + ov->core = ovr_str(two_part[i]); + st = ovr_run(ov, NULL, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok && count == 1 + && !strcmp(ovr_cfg->core_name, "Two Part Test"), + "core lookup: a file id with an underscore", two_part[i]); + } + + /* File names ignore case, display names do not */ + for (i = 0; i < sizeof(other_case) / sizeof(other_case[0]); i++) + { + free(ov->core); + ov->core = ovr_str(other_case[i]); + count = 9; + st = ovr_run(ov, NULL, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok && count == 1 + && !strcmp(ovr_cfg->core_name, "Gamma Test"), + "core lookup: a file name in another case", other_case[i]); + } + free(ov->core); + ov->core = ovr_str("gamma test"); + count = 9; + check( ovr_run(ov, NULL, &count) == MANUAL_CONTENT_SCAN_OVR_NO_SUCH_CORE + && count == 0, + "core lookup: a display name in another case", "no such core"); + + free(ov->core); + ov->core = ovr_str("Slash / Test (Lookup)"); + st = ovr_run(ov, NULL, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok + && !strcmp(ovr_cfg->core_name, "Slash / Test (Lookup)"), + "core lookup: a display name with a slash", ovr_cfg->core_name); + + free(ov->core); + ov->core = ovr_str("nope"); + count = 9; + check( ovr_run(ov, NULL, &count) == MANUAL_CONTENT_SCAN_OVR_NO_SUCH_CORE + && count == 0 + && manual_content_scan_get_menu_core_name(&core_name) + && manual_content_scan_get_scan_method_enum() + == MANUAL_CONTENT_SCAN_METHOD_AUTOMATIC, + "core lookup: unknown", "no such core"); + + /* dup_libretro.so and dup_libretro_android.so share a file id */ + free(ov->core); + ov->core = ovr_str("dup"); + check( ovr_run(ov, NULL, &count) == MANUAL_CONTENT_SCAN_OVR_AMBIGUOUS_CORE + && count == 2, + "core lookup: two files with one file id", "2 cores"); + free(ov->core); + ov->core = ovr_str("dup_libretro.so"); + check( ovr_run(ov, NULL, &count) == MANUAL_CONTENT_SCAN_OVR_AMBIGUOUS_CORE + && count == 2, + "core lookup: and by its file name", "2 cores"); + free(ov->core); + ov->core = ovr_str("dup_libretro_android.so"); + check( ovr_run(ov, NULL, &count) == MANUAL_CONTENT_SCAN_OVR_AMBIGUOUS_CORE + && count == 2, + "core lookup: and by its platform file name", "2 cores"); + strlcpy(dup_path, lookup_core_path("dup_libretro_android.so"), + sizeof(dup_path)); + free(ov->core); + ov->core = ovr_str(dup_path); + st = ovr_run(ov, NULL, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok && count == 1 + && !strcmp(ovr_cfg->core_path, dup_path), + "core lookup: the path tells them apart", path_basename(dup_path)); +#ifdef _WIN32 + /* All '\', '\' before the file name only, and all '\' in capitals */ + for (i = 0; i < 3; i++) + { + char *slash; + + strlcpy(p, dup_path, sizeof(p)); + if (i != 1) + pathname_conform_slashes_to_os(p); + else if ((slash = strrchr(p, '/'))) + *slash = '\\'; + if (i == 2) + string_to_upper(p); + free(ov->core); + ov->core = ovr_str(p); + count = 9; + st = ovr_run(ov, NULL, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok && count == 1 + && !strcmp(ovr_cfg->core_path, dup_path), + "core lookup: the path with either slash", p); + } +#endif + + /* The exact case first, else both are ambiguous; a file system + * that ignores case keeps one of the two */ + twins = *lookup_core_path("twin_libretro.so") + && *lookup_core_path("Twin_libretro.so"); + for (i = 0; i < sizeof(twin) / sizeof(twin[0]); i++) + { + free(ov->core); + ov->core = ovr_str(twin[i][0]); + count = 9; + st = ovr_run(ov, NULL, &count); + if (!twins) + check(st == MANUAL_CONTENT_SCAN_OVR_OK && count == 1, + "core lookup: two file names in another case", "one file"); + else if (*twin[i][1]) + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok && count == 1 + && !strcmp(ovr_cfg->core_path, lookup_core_path(twin[i][1])), + "core lookup: the file name in the case typed", twin[i][0]); + else + check(st == MANUAL_CONTENT_SCAN_OVR_AMBIGUOUS_CORE && count == 2, + "core lookup: two file names in another case", twin[i][0]); + } + + free(ov->core); + ov->core = ovr_str("Same Name Test"); + check( ovr_run(ov, NULL, &count) == MANUAL_CONTENT_SCAN_OVR_AMBIGUOUS_CORE + && count == 2, + "core lookup: two cores with one display name", "2 cores"); + free(ov->core); + ov->core = ovr_str("same2"); + st = ovr_run(ov, NULL, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok && count == 1 + && !strcmp(path_basename(ovr_cfg->core_path), "same2_libretro.so"), + "core lookup: the file name tells them apart", + path_basename(ovr_cfg->core_path)); + + manual_content_scan_overrides_free(ov); + manual_content_scan_reset_settings(); + free(ovr_cfg); + ovr_cfg = NULL; +} + +/* What chain_lane()'s callbacks were handed; chain_in_push, what they + * were before task_push_dbscan_chain() returned. */ +#define CHAIN_SEEN_MAX 8 +static unsigned chain_seen; +static unsigned chain_dones; +static unsigned chain_in_push; +static bool chain_ud_ok; +static char chain_names[CHAIN_SEEN_MAX][512]; +static enum task_dbscan_outcome chain_outcomes[CHAIN_SEEN_MAX]; +static int chain_added[CHAIN_SEEN_MAX]; +/* The system name type each item's scan ran with: the chain puts the + * Manual Scan settings back once it ends. */ +static unsigned chain_name_types[CHAIN_SEEN_MAX]; +static task_dbscan_chain_result_t chain_totals; + +static void chain_item_cb(const char *name, + enum task_dbscan_outcome outcome, + const task_dbscan_result_t *res, void *user_data) +{ + if (user_data != &chain_seen) + chain_ud_ok = false; + if (chain_seen < CHAIN_SEEN_MAX) + { + strlcpy(chain_names[chain_seen], name ? name : "(null)", + sizeof(chain_names[0])); + chain_outcomes[chain_seen] = outcome; + chain_added[chain_seen] = res ? (int)res->added : -1; + chain_name_types[chain_seen] = + manual_content_scan_get_menu_system_name_type(); + } + chain_seen++; +} + +static void chain_done_cb(const task_dbscan_chain_result_t *res, + void *user_data) +{ + if (user_data != &chain_seen) + chain_ud_ok = false; + chain_totals = *res; + chain_dones++; +} + +/* A done callback that starts the next chain: the one ending must no + * longer count as running. */ +static const char *chain_restart_arg; +static bool chain_restart_ok; + +static task_dbscan_item_t *chain_items(size_t n, const char *first, + const char *rest); + +static void chain_done_restart_cb(const task_dbscan_chain_result_t *res, + void *user_data) +{ + chain_done_cb(res, user_data); + chain_restart_ok = task_push_dbscan_chain( + chain_items(1, chain_restart_arg, NULL), 1, + chain_item_cb, chain_done_cb, &chain_seen); +} + +/* A --scan item for @arg; with @core, an override naming it. */ +static void chain_item(task_dbscan_item_t *it, const char *arg, + const char *core) +{ + it->kind = TASK_DBSCAN_ITEM_PATH; + it->arg = ovr_str(arg); + it->overrides = NULL; + if (core && (it->overrides = manual_content_scan_overrides_new())) + it->overrides->core = ovr_str(core); +} + +/* @n items, the first for @first, the rest for @rest. */ +static task_dbscan_item_t *chain_items(size_t n, const char *first, + const char *rest) +{ + size_t i; + task_dbscan_item_t *items = (task_dbscan_item_t*)calloc(n, + sizeof(*items)); + + if (items) + for (i = 0; i < n; i++) + chain_item(&items[i], i ? rest : first, NULL); + return items; +} + +static bool chain_push(task_dbscan_item_t *items, size_t n) +{ + bool pushed; + + chain_seen = 0; + chain_dones = 0; + chain_ud_ok = true; + memset(&chain_totals, 0, sizeof(chain_totals)); + pushed = task_push_dbscan_chain(items, n, chain_item_cb, + chain_done_cb, &chain_seen); + chain_in_push = chain_seen + chain_dones; + return pushed; +} + +/* Runs the queue until done_cb, then a while longer to see it is not + * called again. With @cancel_slow, cancels the scan once a slow read + * has begun, then lets a held one go. */ +static void chain_wait(bool cancel_slow) +{ + task_finder_data_t finder; + unsigned i; + bool cancelled = false; + time_t started = time(NULL); + + finder.func = find_scan; + finder.userdata = NULL; + while (!chain_dones) + { + if ( cancel_slow && !cancelled + && retro_atomic_load_acquire_int(&slow_in) > 0 + && task_queue_find(&finder)) + { + task_queue_cancel_ident(lane_ident); + retro_atomic_store_release_int(&slow_hold, 0); + cancelled = true; + } + task_queue_check(); + if (difftime(time(NULL), started) > SCAN_TIMEOUT_SECONDS) + return; + retro_sleep(1); + } + for (i = 0; i < 20; i++) + { + task_queue_check(); + retro_sleep(1); + } +} + +static bool chain_saw(unsigned i, const char *name, + enum task_dbscan_outcome outcome, int added) +{ + return i < chain_seen && i < CHAIN_SEEN_MAX + && !strcmp(chain_names[i], name) + && chain_outcomes[i] == outcome + && chain_added[i] == added; +} + +/* The chain runs --scan items in order, one task at a time: a missing + * path and an unknown core fail their item, not the run, and the two + * scans of one playlist follow each other, the second pushed from the + * first's callback. A cancelled scan stops the rest. A chain is + * refused while another runs, or a manual scan. */ +static void chain_lane(const char *root, const char *pl_dir, + uint32_t crc_a, uint32_t sz_a, bool threaded) +{ + char a[512], b[512], slow[512], missing[512], pl[512]; + char out[1024], err[1024], p[1024], what[96]; + char detail[160]; + const char *q = threaded ? "(threaded)" : "(regular)"; + task_dbscan_item_t *items; + bool pushed, refused, captured, back, running; + bool scan_refused; + unsigned busy_before, busy_scan; + + sprintf(a, "%s/chain_a", root); + sprintf(b, "%s/chain_b", root); + sprintf(slow, "%s/chain_slow", root); + sprintf(missing, "%s/chain_missing", root); + sprintf(pl, "%s/chain_pl", root); + path_mkdir(a); + path_mkdir(b); + path_mkdir(slow); + path_mkdir(pl); + sprintf(p, "%s/01_alpha.bin", a); + if (!write_content(p, crc_a, sz_a)) + goto fixture; + sprintf(p, "%s/01_alpha.bin", b); + if (!write_content(p, crc_a, sz_a)) + goto fixture; + sprintf(p, "%s/00_slow.bin", slow); + if (!write_content(p, crc_a, sz_a)) + goto fixture; + sprintf(p, "%s/01_more.bin", slow); + if (!write_content(p, 0x3C0FFEE0u, 1024)) + goto fixture; + strlcpy(config_get_ptr()->paths.directory_playlist, pl, + sizeof(config_get_ptr()->paths.directory_playlist)); + delete_playlists(pl); + use_queue(threaded); + + pushed = chain_push(chain_items(1, missing, NULL), 1); + chain_wait(false); + sprintf(what, "chain %s: nothing inside the push", q); + sprintf(detail, "%u calls in the push, done %u", chain_in_push, + chain_dones); + check(pushed && !chain_in_push && chain_dones == 1, what, detail); + sprintf(what, "chain %s: a missing path fails", q); + check( chain_seen == 1 + && chain_saw(0, missing, TASK_DBSCAN_FAILED, -1) + && chain_totals.failed == 1 && !chain_totals.ran + && !chain_totals.added, what, chain_names[0]); + + sprintf(out, "%s/chain_out.txt", root); + sprintf(err, "%s/chain_err.txt", root); + items = (task_dbscan_item_t*)calloc(4, sizeof(*items)); + if (!items) + goto fixture; + chain_item(&items[0], a, NULL); + chain_item(&items[1], missing, NULL); + chain_item(&items[2], b, "nope"); + chain_item(&items[3], b, NULL); + cli_output = true; + captured = capture_begin(out, err); + pushed = chain_push(items, 4); + chain_wait(false); + if (captured) + capture_end(); + cli_output = false; + sprintf(what, "chain %s: items in order", q); + sprintf(detail, "%u items, done %u", chain_seen, chain_dones); + check( captured && pushed && chain_seen == 4 + && chain_saw(0, a, TASK_DBSCAN_RAN, 1) + && chain_saw(1, missing, TASK_DBSCAN_FAILED, -1) + && chain_saw(2, b, TASK_DBSCAN_FAILED, -1) + && chain_saw(3, b, TASK_DBSCAN_RAN, 1), what, detail); + sprintf(what, "chain %s: the totals", q); + sprintf(detail, "ran %u, added %u, failed %u, skipped %u%s", + chain_totals.ran, chain_totals.added, chain_totals.failed, + chain_totals.skipped, chain_totals.cancelled ? ", cancelled" : ""); + check( chain_totals.ran == 2 && chain_totals.added == 2 + && chain_totals.failed == 2 && !chain_totals.skipped + && !chain_totals.cancelled, what, detail); + sprintf(what, "chain %s: done_cb once, with the user data", q); + check(chain_dones == 1 && chain_ud_ok && !chain_in_push, what, ""); + sprintf(what, "chain %s: one playlist twice in a row", q); + sprintf(p, "%s/Test Alpha.lpl", pl); + { + char ea[1024], eb[1024]; + + sprintf(ea, "%s/01_alpha.bin", a); + sprintf(eb, "%s/01_alpha.bin", b); + check(file_contains(p, ea) && file_contains(p, eb), what, p); + } + sprintf(what, "chain %s: the item lines", q); + sprintf(p, "%s: not found", missing); + check( file_contains(err, p) + && file_contains(err, "nope: no such core"), what, err); + + /* Held on the threaded queue, so the scan cannot end before the + * cancel; the regular one reads on this thread */ + slow_reset(sz_a, 300); + if (threaded) + retro_atomic_store_release_int(&slow_hold, 1); + pushed = chain_push(chain_items(2, slow, a), 2); + chain_wait(true); + retro_atomic_store_release_int(&slow_hold, 0); + slow_size = 0; + back = slow_reads_back() && readers_gone(); + sprintf(what, "chain %s: a cancelled scan stops the rest", q); + sprintf(detail, "%u items, ran %u%s, done %u", chain_seen, + chain_totals.ran, chain_totals.cancelled ? ", cancelled" : "", + chain_dones); + check( pushed && back && chain_seen == 1 && chain_dones == 1 + && chain_saw(0, slow, TASK_DBSCAN_RAN, chain_added[0]) + && chain_totals.ran == 1 && chain_totals.cancelled, what, detail); + + /* what the menu holds before the chain, to find again after it */ + manual_content_scan_reset_settings(); + strlcpy(manual_content_scan_get_system_name_custom_ptr(), "Kept Name", + manual_content_scan_get_system_name_custom_size()); + pushed = chain_push(chain_items(1, a, NULL), 1); + running = task_dbscan_chain_is_running(); + refused = !task_push_dbscan_chain(chain_items(1, b, NULL), 1, + chain_item_cb, chain_done_cb, &chain_seen); + busy_before = busy_pushed; + scan_refused = !task_push_dbscan(b, scan_cb, NULL); + busy_scan = busy_pushed - busy_before; + chain_wait(false); + sprintf(what, "chain %s: a second chain refused", q); + sprintf(detail, "%u items, done %u", chain_seen, chain_dones); + check( pushed && refused && chain_seen == 1 && chain_dones == 1 + && chain_saw(0, a, TASK_DBSCAN_RAN, 0), what, detail); + sprintf(what, "chain %s: running from the push to the end", q); + check(running && !task_dbscan_chain_is_running(), what, ""); + sprintf(what, "chain %s: a scan refused while it runs says why", q); + sprintf(detail, "refused %d, told %u", (int)scan_refused, busy_scan); + check(scan_refused && busy_scan == 1, what, detail); + sprintf(what, "chain %s: the Manual Scan settings come back", q); + check(!strcmp(manual_content_scan_get_system_name_custom_ptr(), + "Kept Name"), what, + manual_content_scan_get_system_name_custom_ptr()); + manual_content_scan_reset_settings(); + + /* the done callback starts the next chain */ + chain_seen = 0; + chain_dones = 0; + chain_ud_ok = true; + chain_restart_ok = false; + chain_restart_arg = b; + pushed = task_push_dbscan_chain(chain_items(1, a, NULL), 1, + chain_item_cb, chain_done_restart_cb, &chain_seen); + { + time_t started = time(NULL); + while ( pushed && chain_dones < 2 + && difftime(time(NULL), started) <= SCAN_TIMEOUT_SECONDS) + { + task_queue_check(); + retro_sleep(1); + } + } + sprintf(what, "chain %s: its done callback may start another", q); + sprintf(detail, "pushed %d, restarted %d, done %u, items %u", + (int)pushed, (int)chain_restart_ok, chain_dones, chain_seen); + check( pushed && chain_restart_ok && chain_dones == 2 + && chain_saw(0, a, TASK_DBSCAN_RAN, 0) + && chain_saw(1, b, TASK_DBSCAN_RAN, chain_added[1]), + what, detail); + + /* Held, so the scan cannot end before the chain is pushed */ + slow_reset(sz_a, 0); + retro_atomic_store_release_int(&slow_hold, 1); + loop_active = true; + scan_completed = false; + pushed = task_push_dbscan(a, scan_cb, NULL); + refused = !chain_push(chain_items(1, b, NULL), 1); + retro_atomic_store_release_int(&slow_hold, 0); + { + time_t started = time(NULL); + + while (loop_active && difftime(time(NULL), started) + <= SCAN_TIMEOUT_SECONDS) + { + task_queue_check(); + retro_sleep(1); + } + } + slow_size = 0; + sprintf(what, "chain %s: refused while a manual scan runs", q); + check( pushed && refused && scan_completed && !chain_seen + && !chain_dones, what, refused ? "refused" : "pushed"); + sprintf(what, "chain %s: no task_push_dbscan while it runs", q); + check(scan_refused && pushed, what, scan_refused ? "refused" : "pushed"); + + manual_content_scan_reset_settings(); + strlcpy(config_get_ptr()->paths.directory_playlist, pl_dir, + sizeof(config_get_ptr()->paths.directory_playlist)); + return; + +fixture: + check(0, "fixture", "could not write the chain lane"); + strlcpy(config_get_ptr()->paths.directory_playlist, pl_dir, + sizeof(config_get_ptr()->paths.directory_playlist)); +} + +/* Linked with --wrap: how many database listings, playlist-folder + * walks and playlist loads began, and how many of them on the main + * thread. */ +enum { REFRESH_RDB = 0, REFRESH_LPL, REFRESH_LOAD, REFRESH_KINDS }; +static retro_atomic_int_t refresh_began[REFRESH_KINDS]; +static retro_atomic_int_t refresh_began_main[REFRESH_KINDS]; + +static void refresh_began_reset(void) +{ + int k; + + for (k = 0; k < REFRESH_KINDS; k++) + { + retro_atomic_store_release_int(&refresh_began[k], 0); + retro_atomic_store_release_int(&refresh_began_main[k], 0); + } +} + +static void refresh_began_one(int k) +{ + retro_atomic_fetch_add_int(&refresh_began[k], 1); + if (task_is_on_main_thread()) + retro_atomic_fetch_add_int(&refresh_began_main[k], 1); +} + +/* Set by the walk's own thread, which is the one that steps it. */ +static retro_atomic_int_t lpl_walk_fail; +static dir_list_iter_t *lpl_fail_iter; + +dir_list_iter_t *__real_dir_list_iter_new(const char *dir, const char *ext, + bool include_dirs, bool include_hidden, bool include_compressed, + bool recursive, struct string_list *list); +dir_list_iter_t *__wrap_dir_list_iter_new(const char *dir, const char *ext, + bool include_dirs, bool include_hidden, bool include_compressed, + bool recursive, struct string_list *list) +{ + dir_list_iter_t *iter; + if (ext && !strcmp(ext, "rdb")) + refresh_began_one(REFRESH_RDB); + else if (ext && !strcmp(ext, "lpl")) + { + refresh_began_one(REFRESH_LPL); + prep_cancel_begun(); + } + iter = __real_dir_list_iter_new(dir, ext, include_dirs, include_hidden, + include_compressed, recursive, list); + if ( ext && !strcmp(ext, "lpl") + && retro_atomic_load_acquire_int(&lpl_walk_fail)) + lpl_fail_iter = iter; + return iter; +} + +/* Linked with --wrap: the playlist folder walk fails as out of + * memory does. */ +int __real_dir_list_iter_step(dir_list_iter_t *iter, + bool (*within_budget)(void *ud), void *ud); +int __wrap_dir_list_iter_step(dir_list_iter_t *iter, + bool (*within_budget)(void *ud), void *ud) +{ + if (iter && iter == lpl_fail_iter) + return -1; + return __real_dir_list_iter_step(iter, within_budget, ud); +} + +/* Linked with --wrap: a refresh's load reads the playlist's header. */ +playlist_parse_t *__real_playlist_parse_begin_header( + const playlist_config_t *config); +playlist_parse_t *__wrap_playlist_parse_begin_header( + const playlist_config_t *config) +{ + refresh_began_one(REFRESH_LOAD); + prep_cancel_begun(); + return __real_playlist_parse_begin_header(config); +} + +/* Linked with --wrap: the task_init() this counts down to fails, as + * out of memory. */ +static retro_atomic_int_t task_init_fail_in; + +retro_task_t *__real_task_init(void); +retro_task_t *__wrap_task_init(void) +{ + if ( retro_atomic_load_acquire_int(&task_init_fail_in) > 0 + && retro_atomic_fetch_add_int(&task_init_fail_in, -1) == 1) + return NULL; + return __real_task_init(); +} + +/* Writes @lpl with one entry, @entry, and unless @dir is NULL a scan + * record for @dir with the saved core @core (NULL: detect). */ +static bool refresh_write(const char *lpl, const char *dir, + const char *core, const char *entry) +{ + playlist_config_t *cfg = (playlist_config_t*)calloc(1, sizeof(*cfg)); + playlist_t *pl = NULL; + struct playlist_entry e; + bool ok = false; + + if (!cfg) + return false; + playlist_config_set_path(cfg, lpl); + cfg->capacity = COLLECTION_SIZE; + if ((pl = playlist_init(cfg))) + { + memset(&e, 0, sizeof(e)); + e.path = (char*)entry; + e.label = (char*)"Stale"; + e.core_path = (char*)"DETECT"; + e.core_name = (char*)"DETECT"; + ok = playlist_push(pl, &e); + if (dir) + { + playlist_set_scan_content_dir(pl, dir); + playlist_set_scan_search_recursively(pl, true); + if (core) + playlist_set_default_core_name(pl, core); + } + playlist_write_file(pl); + playlist_free(pl); + ok = ok && path_is_valid(lpl); + } + free(cfg); + return ok; +} + +static void refresh_item(task_dbscan_item_t *it, + enum task_dbscan_item_kind kind, const char *arg) +{ + it->kind = kind; + it->arg = arg ? ovr_str(arg) : NULL; + it->overrides = NULL; +} + +/* Runs @items with stdout in @out and stderr in @err; @cli as for a + * --scan run. */ +static bool refresh_run(task_dbscan_item_t *items, size_t n, + const char *out, const char *err, bool cli) +{ + bool captured, pushed; + + cli_output = cli; + captured = capture_begin(out, err); + pushed = chain_push(items, n); + chain_wait(false); + if (captured) + capture_end(); + cli_output = false; + return captured && pushed && chain_dones == 1; +} + +/* True if @dir lists @name in that case; @detail gets what it lists. */ +static bool listed_as(const char *dir, const char *name, char *detail, + size_t size) +{ + size_t i; + bool found = false; + struct string_list *l = dir_list_new(dir, "lpl", false, false, + false, false); + + *detail = '\0'; + for (i = 0; l && i < l->size; i++) + { + const char *base = path_basename(l->elems[i].data); + + if (i) + strlcat(detail, " ", size); + strlcat(detail, base, size); + if (string_is_equal(base, name)) + found = true; + } + string_list_free(l); + return found; +} + +/* Refresh items over a playlist folder: a refresh-all takes every + * playlist with saved settings in name order, history and favorites + * never, skips one whose folder is gone and fails one whose core is; + * it refreshes a playlist an earlier item wrote. A refresh names its + * playlist by name or by a path in the playlist folder. The walks and + * loads run on the worker when the queue has one. */ +static void refresh_lane(const char *root, const char *pl_dir, + uint32_t crc_a, uint32_t sz_a, bool threaded) +{ + char pl[512], none_pl[512], a[512], gone[512]; + char fresh[512], other[512], case_pl[512]; + char dup_pl[512]; + char out[1024], err[1024]; + char p[1024], stale[1024], want[1536], what[128]; + char detail[160]; + const char *q = threaded ? "(threaded)" : "(regular)"; + task_dbscan_item_t *items; + bool ran, off_main = false; + +#ifdef HAVE_THREADS + off_main = threaded; +#endif + sprintf(pl, "%s/refresh_pl", root); + sprintf(none_pl, "%s/refresh_none", root); + sprintf(a, "%s/refresh_a", root); + sprintf(gone, "%s/refresh_gone", root); + sprintf(fresh, "%s/refresh_new", root); + sprintf(other, "%s/refresh_other", root); + sprintf(case_pl, "%s/refresh_case", root); + sprintf(dup_pl, "%s/refresh_dup", root); + sprintf(out, "%s/refresh_out.txt", root); + sprintf(err, "%s/refresh_err.txt", root); + sprintf(stale, "%s/00_gone.bin", a); + path_mkdir(pl); + path_mkdir(none_pl); + path_mkdir(a); + path_mkdir(fresh); + path_mkdir(other); + path_mkdir(case_pl); + path_mkdir(dup_pl); + delete_playlists(pl); + delete_playlists(none_pl); + delete_playlists(other); + delete_playlists(case_pl); + delete_playlists(dup_pl); + sprintf(p, "%s/01_alpha.bin", a); + if (!write_content(p, crc_a, sz_a)) + goto fixture; + sprintf(p, "%s/01_alpha.bin", fresh); + if (!write_content(p, crc_a, sz_a)) + goto fixture; + sprintf(p, "%s/a_ok.lpl", pl); + if (!refresh_write(p, a, NULL, stale)) + goto fixture; + sprintf(p, "%s/b_gone.lpl", pl); + if (!refresh_write(p, gone, NULL, stale)) + goto fixture; + sprintf(p, "%s/c_none.lpl", pl); + if (!refresh_write(p, NULL, NULL, stale)) + goto fixture; + sprintf(p, "%s/e_core.lpl", pl); + if (!refresh_write(p, a, "No Such Core", stale)) + goto fixture; + sprintf(p, "%s/x_history.lpl", pl); + if (!refresh_write(p, a, NULL, stale)) + goto fixture; + sprintf(p, "%s/content_favorites.lpl", pl); + if (!refresh_write(p, a, NULL, stale)) + goto fixture; + sprintf(p, "%s/c_none.lpl", none_pl); + if (!refresh_write(p, NULL, NULL, stale)) + goto fixture; + sprintf(p, "%s/x_history.lpl", none_pl); + if (!refresh_write(p, a, NULL, stale)) + goto fixture; + sprintf(p, "%s/a_ok.lpl", other); + if (!refresh_write(p, a, NULL, stale)) + goto fixture; + strlcpy(config_get_ptr()->paths.directory_playlist, pl, + sizeof(config_get_ptr()->paths.directory_playlist)); + use_queue(threaded); + + /* A --scan writing n_new, then the refresh-all. */ + if (!(items = (task_dbscan_item_t*)calloc(2, sizeof(*items)))) + goto fixture; + chain_item(&items[0], fresh, NULL); + if ((items[0].overrides = manual_content_scan_overrides_new())) + items[0].overrides->playlist = ovr_str("n_new"); + refresh_item(&items[1], TASK_DBSCAN_ITEM_REFRESH_ALL, NULL); + refresh_began_reset(); + ran = refresh_run(items, 2, out, err, true); + sprintf(what, "refresh %s: refresh-all in name order", q); + snprintf(detail, sizeof(detail), "%u items: %s, %s, %s, %s", chain_seen, + chain_names[1], chain_names[2], chain_names[3], chain_names[4]); + check( ran && chain_seen == 5 + && chain_saw(0, fresh, TASK_DBSCAN_RAN, 1) + && chain_saw(1, "a_ok", TASK_DBSCAN_RAN, 1) + && chain_saw(2, "b_gone", TASK_DBSCAN_SKIPPED, -1) + && chain_saw(3, "e_core", TASK_DBSCAN_FAILED, -1) + && chain_saw(4, "n_new", TASK_DBSCAN_RAN, 0), what, detail); + sprintf(what, "refresh %s: the totals", q); + sprintf(detail, "ran %u, added %u, failed %u, skipped %u%s", + chain_totals.ran, chain_totals.added, chain_totals.failed, + chain_totals.skipped, chain_totals.cancelled ? ", cancelled" : ""); + check( chain_totals.ran == 3 && chain_totals.added == 2 + && chain_totals.failed == 1 && chain_totals.skipped == 1 + && !chain_totals.cancelled, what, detail); + sprintf(what, "refresh %s: a stale entry removed", q); + sprintf(p, "%s/a_ok.lpl", pl); + sprintf(want, "%s/01_alpha.bin", a); + check(file_contains(p, want) && !file_contains(p, stale), what, p); + sprintf(what, "refresh %s: history and favorites never loaded", q); + sprintf(p, "%s/x_history.lpl", pl); + sprintf(detail, "%d loads", + retro_atomic_load_acquire_int(&refresh_began[REFRESH_LOAD])); + check( file_contains(p, stale) && !file_contains(p, want) + && retro_atomic_load_acquire_int(&refresh_began[REFRESH_LOAD]) == 5, + what, detail); + sprintf(what, "refresh %s: the refresh lines", q); + sprintf(want, "%s: skipped: content folder not found (%s)", "b_gone", + gone); + check( file_contains(out, + "a_ok: 1 added, 0 duplicates, 0 unmatched, 0 errors") + && file_contains(err, want) + && file_contains(err, "e_core: saved core not found (No Such Core)") + && !file_contains(out, "b_gone") + && !file_contains(err, "no playlists"), what, err); + sprintf(what, "refresh %s: walks and loads %s", q, + off_main ? "off the main thread" : "ran"); + sprintf(detail, "rdb %d (%d main), lpl %d (%d main), loads %d (%d main)", + retro_atomic_load_acquire_int(&refresh_began[REFRESH_RDB]), + retro_atomic_load_acquire_int(&refresh_began_main[REFRESH_RDB]), + retro_atomic_load_acquire_int(&refresh_began[REFRESH_LPL]), + retro_atomic_load_acquire_int(&refresh_began_main[REFRESH_LPL]), + retro_atomic_load_acquire_int(&refresh_began[REFRESH_LOAD]), + retro_atomic_load_acquire_int(&refresh_began_main[REFRESH_LOAD])); + check( retro_atomic_load_acquire_int(&refresh_began[REFRESH_RDB]) >= 1 + && retro_atomic_load_acquire_int(&refresh_began[REFRESH_LPL]) == 1 + && (!off_main || ( + !retro_atomic_load_acquire_int(&refresh_began_main[REFRESH_RDB]) + && !retro_atomic_load_acquire_int(&refresh_began_main[REFRESH_LPL]) + && !retro_atomic_load_acquire_int( + &refresh_began_main[REFRESH_LOAD]))), what, detail); + + /* By name with an override, by path, and the naming failures. */ + if (!(items = (task_dbscan_item_t*)calloc(5, sizeof(*items)))) + goto fixture; + refresh_item(&items[0], TASK_DBSCAN_ITEM_REFRESH, "a_ok"); + if ((items[0].overrides = manual_content_scan_overrides_new())) + items[0].overrides->exts = ovr_str("bin"); + sprintf(p, "%s/a_ok.lpl", pl); + refresh_item(&items[1], TASK_DBSCAN_ITEM_REFRESH, p); + sprintf(p, "%s/a_ok.lpl", other); + refresh_item(&items[2], TASK_DBSCAN_ITEM_REFRESH, p); + refresh_item(&items[3], TASK_DBSCAN_ITEM_REFRESH, "nope"); + refresh_item(&items[4], TASK_DBSCAN_ITEM_REFRESH, "c_none"); + ran = refresh_run(items, 5, out, err, true); + sprintf(what, "refresh %s: by name, by path, and what fails", q); + sprintf(detail, "%u items, ran %u, failed %u", chain_seen, + chain_totals.ran, chain_totals.failed); + check( ran && chain_seen == 5 + && chain_saw(0, "a_ok", TASK_DBSCAN_RAN, 0) + && chain_saw(1, "a_ok", TASK_DBSCAN_RAN, 0) + && chain_saw(2, p, TASK_DBSCAN_FAILED, -1) + && chain_saw(3, "nope", TASK_DBSCAN_FAILED, -1) + && chain_saw(4, "c_none", TASK_DBSCAN_FAILED, -1) + && chain_totals.ran == 2 && chain_totals.failed == 3 + && !chain_totals.added, what, detail); + sprintf(what, "refresh %s: an override written back", q); + sprintf(want, "%s/a_ok.lpl", pl); + check(file_contains(want, "\"scan_file_exts\": \"bin\""), what, want); + sprintf(what, "refresh %s: the naming lines", q); + sprintf(want, "%s: not in the playlist folder (%s)", p, pl); + check( file_contains(err, want) + && file_contains(err, "nope: no such playlist") + && file_contains(err, "c_none: no saved scan settings"), what, err); + + /* A file name without a folder is the playlist folder's. */ + if (!(items = (task_dbscan_item_t*)calloc(2, sizeof(*items)))) + goto fixture; + refresh_item(&items[0], TASK_DBSCAN_ITEM_REFRESH, "a_ok.lpl"); + refresh_item(&items[1], TASK_DBSCAN_ITEM_REFRESH, "A_OK.LPL"); + ran = refresh_run(items, 2, out, err, true); + sprintf(what, "refresh %s: a file name, not a path", q); + check( ran && chain_saw(0, "a_ok", TASK_DBSCAN_RAN, 0) + && chain_saw(1, "a_ok", TASK_DBSCAN_RAN, 0) + && path_get_size(err) == 0, what, err); + +#ifdef _WIN32 + /* All '\' and '\' before the file name only. */ + if (!(items = (task_dbscan_item_t*)calloc(2, sizeof(*items)))) + goto fixture; + sprintf(p, "%s/a_ok.lpl", pl); + pathname_conform_slashes_to_os(p); + refresh_item(&items[0], TASK_DBSCAN_ITEM_REFRESH, p); + sprintf(p, "%s\\a_ok.lpl", pl); + refresh_item(&items[1], TASK_DBSCAN_ITEM_REFRESH, p); + ran = refresh_run(items, 2, out, err, true); + sprintf(what, "refresh %s: a path with either slash", q); + sprintf(detail, "%u items, ran %u, failed %u", chain_seen, + chain_totals.ran, chain_totals.failed); + check( ran && chain_seen == 2 + && chain_saw(0, "a_ok", TASK_DBSCAN_RAN, 0) + && chain_saw(1, "a_ok", TASK_DBSCAN_RAN, 0) + && chain_totals.ran == 2 && !chain_totals.failed, what, detail); + + /* A playlist folder with '\', a path with '/'. */ + strlcpy(p, pl, sizeof(p)); + pathname_conform_slashes_to_os(p); + strlcpy(config_get_ptr()->paths.directory_playlist, p, + sizeof(config_get_ptr()->paths.directory_playlist)); + if (!(items = (task_dbscan_item_t*)calloc(2, sizeof(*items)))) + goto fixture; + refresh_item(&items[0], TASK_DBSCAN_ITEM_REFRESH, "a_ok"); + sprintf(p, "%s/a_ok.lpl", pl); + refresh_item(&items[1], TASK_DBSCAN_ITEM_REFRESH, p); + ran = refresh_run(items, 2, out, err, true); + strlcpy(config_get_ptr()->paths.directory_playlist, pl, + sizeof(config_get_ptr()->paths.directory_playlist)); + sprintf(what, "refresh %s: a playlist folder with '\\'", q); + sprintf(detail, "%u items, ran %u, failed %u", chain_seen, + chain_totals.ran, chain_totals.failed); + check( ran && chain_seen == 2 + && chain_saw(0, "a_ok", TASK_DBSCAN_RAN, 0) + && chain_saw(1, "a_ok", TASK_DBSCAN_RAN, 0) + && chain_totals.ran == 2 && !chain_totals.failed, what, detail); +#endif + + /* Another case: the lines, the system name and the save take the + * file's own name, so a database's playlist stays one. */ + sprintf(p, "%s/Test Beta.lpl", case_pl); + if (!refresh_write(p, a, NULL, stale)) + goto fixture; + strlcpy(config_get_ptr()->paths.directory_playlist, case_pl, + sizeof(config_get_ptr()->paths.directory_playlist)); + if (!(items = (task_dbscan_item_t*)calloc(2, sizeof(*items)))) + goto fixture; + refresh_item(&items[0], TASK_DBSCAN_ITEM_REFRESH, "test beta"); + sprintf(p, "%s/TEST BETA.LPL", case_pl); + refresh_item(&items[1], TASK_DBSCAN_ITEM_REFRESH, p); + ran = refresh_run(items, 2, out, err, true); + strlcpy(config_get_ptr()->paths.directory_playlist, pl, + sizeof(config_get_ptr()->paths.directory_playlist)); + sprintf(what, "refresh %s: another case, named as the file", q); + snprintf(detail, sizeof(detail), "%u items: %s, %s", chain_seen, + chain_names[0], chain_names[1]); + check( ran && chain_seen == 2 + && !strcmp(chain_names[0], "Test Beta") + && !strcmp(chain_names[1], "Test Beta") + && chain_outcomes[0] == TASK_DBSCAN_RAN + && chain_outcomes[1] == TASK_DBSCAN_RAN + && file_contains(out, "Test Beta: ") + && !file_contains(out, "test beta") + && !file_contains(out, "TEST BETA"), what, detail); + sprintf(what, "refresh %s: another case, a database's playlist", q); + check( chain_seen == 2 + && chain_name_types[0] == MANUAL_CONTENT_SCAN_SYSTEM_NAME_DATABASE + && chain_name_types[1] == MANUAL_CONTENT_SCAN_SYSTEM_NAME_DATABASE, + what, ""); + sprintf(what, "refresh %s: another case keeps the file's name", q); + check( listed_as(case_pl, "Test Beta.lpl", detail, sizeof(detail)) + && !strcmp(detail, "Test Beta.lpl"), what, detail); + + /* Two files in other cases: neither is the name; the exact one is. */ + sprintf(p, "%s/Test Beta.lpl", dup_pl); + if (!refresh_write(p, a, NULL, stale)) + goto fixture; + sprintf(p, "%s/TEST BETA.lpl", dup_pl); + /* Where case is ignored this is the same file, already written. */ + refresh_write(p, a, NULL, stale); + if ( listed_as(dup_pl, "Test Beta.lpl", detail, sizeof(detail)) + && listed_as(dup_pl, "TEST BETA.lpl", detail, sizeof(detail))) + { + strlcpy(config_get_ptr()->paths.directory_playlist, dup_pl, + sizeof(config_get_ptr()->paths.directory_playlist)); + if (!(items = (task_dbscan_item_t*)calloc(2, sizeof(*items)))) + goto fixture; + refresh_item(&items[0], TASK_DBSCAN_ITEM_REFRESH, "test beta"); + refresh_item(&items[1], TASK_DBSCAN_ITEM_REFRESH, "Test Beta"); + ran = refresh_run(items, 2, out, err, true); + strlcpy(config_get_ptr()->paths.directory_playlist, pl, + sizeof(config_get_ptr()->paths.directory_playlist)); + sprintf(what, "refresh %s: two in other cases, ambiguous", q); + check( ran && chain_saw(0, "test beta", TASK_DBSCAN_FAILED, -1) + && file_contains(err, + "test beta: 2 playlists with this name in other cases, " + "give the exact name\n"), + what, err); + sprintf(what, "refresh %s: two in other cases, the exact one", q); + snprintf(detail, sizeof(detail), "%u items, ran %u, failed %u", + chain_seen, chain_totals.ran, chain_totals.failed); + check( ran && chain_seen == 2 + && !strcmp(chain_names[1], "Test Beta") + && chain_outcomes[1] == TASK_DBSCAN_RAN + && chain_totals.ran == 1 && chain_totals.failed == 1, + what, detail); + } + else + printf(" skip refresh %s: two in other cases (the file system " + "ignores case: %s)\n", q, detail); + + /* Nothing to refresh. */ + strlcpy(config_get_ptr()->paths.directory_playlist, none_pl, + sizeof(config_get_ptr()->paths.directory_playlist)); + if (!(items = (task_dbscan_item_t*)calloc(1, sizeof(*items)))) + goto fixture; + refresh_item(&items[0], TASK_DBSCAN_ITEM_REFRESH_ALL, NULL); + ran = refresh_run(items, 1, out, err, true); + sprintf(what, "refresh %s: a refresh-all with nothing to refresh", q); + check( ran && !chain_seen && !chain_totals.ran && !chain_totals.failed + && !chain_totals.skipped + && file_contains(err, "no playlists with saved scan settings"), + what, err); + + /* The second task_init is the refresh-all's listing. */ + if (!(items = (task_dbscan_item_t*)calloc(1, sizeof(*items)))) + goto fixture; + refresh_item(&items[0], TASK_DBSCAN_ITEM_REFRESH_ALL, NULL); + retro_atomic_store_release_int(&task_init_fail_in, 2); + ran = refresh_run(items, 1, out, err, true); + retro_atomic_store_release_int(&task_init_fail_in, 0); + sprintf(what, "refresh %s: a refresh-all out of memory", q); + check( ran && chain_saw(0, none_pl, TASK_DBSCAN_FAILED, -1) + && chain_totals.failed == 1 && !chain_totals.ran + && file_contains(err, "out of memory"), what, err); + + /* The listing fails part way. */ + if (!(items = (task_dbscan_item_t*)calloc(1, sizeof(*items)))) + goto fixture; + refresh_item(&items[0], TASK_DBSCAN_ITEM_REFRESH_ALL, NULL); + retro_atomic_store_release_int(&lpl_walk_fail, 1); + ran = refresh_run(items, 1, out, err, true); + retro_atomic_store_release_int(&lpl_walk_fail, 0); + lpl_fail_iter = NULL; + sprintf(what, "refresh %s: a refresh-all whose listing fails", q); + check( ran && chain_saw(0, none_pl, TASK_DBSCAN_FAILED, -1) + && chain_totals.failed == 1 && !chain_totals.ran + && file_contains(err, "out of memory") + && !file_contains(err, "no playlists with saved scan settings"), + what, err); + + /* Cancelled while the folder is walked, then while its second + * playlist loads: nothing after runs. */ + strlcpy(config_get_ptr()->paths.directory_playlist, pl, + sizeof(config_get_ptr()->paths.directory_playlist)); + if (!(items = (task_dbscan_item_t*)calloc(2, sizeof(*items)))) + goto fixture; + refresh_item(&items[0], TASK_DBSCAN_ITEM_REFRESH_ALL, NULL); + chain_item(&items[1], fresh, NULL); + prep_pushes = 0; + prep_cancel_at = 2; + ran = refresh_run(items, 2, out, err, true); + prep_cancel_at = 0; + sprintf(what, "refresh %s: cancelled while walking the folder", q); + sprintf(detail, "%u items, ran %u, done %u%s", chain_seen, + chain_totals.ran, chain_dones, + chain_totals.cancelled ? ", cancelled" : ""); + check( ran && !prep_cancel_task && !chain_seen + && chain_totals.cancelled && !chain_totals.ran + && !chain_totals.failed, what, detail); + if (!(items = (task_dbscan_item_t*)calloc(2, sizeof(*items)))) + goto fixture; + refresh_item(&items[0], TASK_DBSCAN_ITEM_REFRESH_ALL, NULL); + chain_item(&items[1], fresh, NULL); + prep_pushes = 0; + prep_cancel_at = 4; /* the databases, the walk, a_ok, b_gone */ + ran = refresh_run(items, 2, out, err, true); + prep_cancel_at = 0; + sprintf(what, "refresh %s: cancelled while loading a playlist", q); + sprintf(detail, "%u items: %s, ran %u, done %u%s", chain_seen, + chain_names[0], chain_totals.ran, chain_dones, + chain_totals.cancelled ? ", cancelled" : ""); + check( ran && !prep_cancel_task && chain_seen == 1 + && !strcmp(chain_names[0], "a_ok") + && chain_outcomes[0] == TASK_DBSCAN_RAN + && chain_totals.cancelled && chain_totals.ran == 1 + && !chain_totals.skipped && !chain_totals.failed, what, detail); + + /* Without --scan: the outcome, no lines. */ + if (!(items = (task_dbscan_item_t*)calloc(2, sizeof(*items)))) + goto fixture; + refresh_item(&items[0], TASK_DBSCAN_ITEM_REFRESH, "b_gone"); + refresh_item(&items[1], TASK_DBSCAN_ITEM_REFRESH, "a_ok"); + ran = refresh_run(items, 2, out, err, false); + sprintf(what, "refresh %s: nothing prints without --scan", q); + check( ran && chain_saw(0, "b_gone", TASK_DBSCAN_SKIPPED, -1) + && chain_saw(1, "a_ok", TASK_DBSCAN_RAN, 0) + && path_get_size(out) == 0 && path_get_size(err) == 0, what, err); + + manual_content_scan_reset_settings(); + strlcpy(config_get_ptr()->paths.directory_playlist, pl_dir, + sizeof(config_get_ptr()->paths.directory_playlist)); + return; + +fixture: + check(0, "fixture", "could not write the refresh lane"); + strlcpy(config_get_ptr()->paths.directory_playlist, pl_dir, + sizeof(config_get_ptr()->paths.directory_playlist)); +} + +/* File system calls on the main thread while fs_main_watch is set: + * the scan chain's must all run in its prep tasks. Only the main + * thread reads the flag. */ +static bool fs_main_watch; +static unsigned fs_main_calls; +static char fs_main_last[64]; + +static void fs_main_note(const char *fn) +{ +#ifdef HAVE_THREADS + if (current_tid() == main_tid && fs_main_watch) + { + fs_main_calls++; + strlcpy(fs_main_last, fn, sizeof(fs_main_last)); + } +#else + (void)fn; +#endif +} + +int __real_path_stat(const char *path); +int __wrap_path_stat(const char *path) +{ + fs_main_note("path_stat"); + return __real_path_stat(path); +} + +bool __real_path_is_valid(const char *path); +bool __wrap_path_is_valid(const char *path) +{ + fs_main_note("path_is_valid"); + return __real_path_is_valid(path); +} + +bool __real_path_is_directory(const char *path); +bool __wrap_path_is_directory(const char *path) +{ + fs_main_note("path_is_directory"); + return __real_path_is_directory(path); +} + +int64_t __real_path_get_size(const char *path); +int64_t __wrap_path_get_size(const char *path) +{ + fs_main_note("path_get_size"); + return __real_path_get_size(path); +} + +char *__real_path_resolve_realpath(char *s, size_t len, + bool resolve_symlinks); +char *__wrap_path_resolve_realpath(char *s, size_t len, + bool resolve_symlinks) +{ + fs_main_note("path_resolve_realpath"); + return __real_path_resolve_realpath(s, len, resolve_symlinks); +} + +/* On a threaded queue the chain asks the file system nothing on the + * main thread before a scan: a --scan path's resolve and stat, an + * override core path's and DAT's, a --scan-refresh path's realpaths + * and its content folder's stat all run on the worker. Every item + * here stops before its scan, so the whole run is watched. */ +static void fs_main_lane(const char *root, const char *pl_dir) +{ +#ifdef HAVE_THREADS + char pl[512], a[512], gone[512], outside[512]; + char by_path[1024], p[1024], stale[1024], detail[256]; + task_dbscan_item_t *items; + bool pushed; + + sprintf(pl, "%s/fsmain_pl", root); + sprintf(a, "%s/fsmain_a", root); + sprintf(gone, "%s/fsmain_gone", root); + sprintf(outside, "%s/fsmain_out", root); + sprintf(stale, "%s/00_gone.bin", a); + path_mkdir(pl); + path_mkdir(a); + path_mkdir(outside); + delete_playlists(pl); + sprintf(p, "%s/f_gone.lpl", pl); + if (!refresh_write(p, gone, NULL, stale)) + goto fixture; + sprintf(p, "%s/f_none.lpl", pl); + if (!refresh_write(p, NULL, NULL, stale)) + goto fixture; + sprintf(p, "%s/x.lpl", outside); + if (!refresh_write(p, a, NULL, stale)) + goto fixture; + /* By its path: the check that it is in the folder realpaths both */ + sprintf(by_path, "%s/f_gone.lpl", pl); + strlcpy(config_get_ptr()->paths.directory_playlist, pl, + sizeof(config_get_ptr()->paths.directory_playlist)); + use_queue(true); + + if (!(items = (task_dbscan_item_t*)calloc(6, sizeof(*items)))) + goto fixture; + chain_item(&items[0], "fsmain_missing_dir", NULL); + chain_item(&items[1], a, "./fsmain_nocore_libretro.so"); + chain_item(&items[2], a, NULL); + if ((items[2].overrides = manual_content_scan_overrides_new())) + items[2].overrides->dat = ovr_str("fsmain_missing.dat"); + refresh_item(&items[3], TASK_DBSCAN_ITEM_REFRESH, by_path); + refresh_item(&items[4], TASK_DBSCAN_ITEM_REFRESH, p); + refresh_item(&items[5], TASK_DBSCAN_ITEM_REFRESH, "f_none"); + + fs_main_calls = 0; + fs_main_last[0] = '\0'; + fs_main_watch = true; + pushed = chain_push(items, 6); + chain_wait(false); + fs_main_watch = false; + + snprintf(detail, sizeof(detail), "%u items, failed %u, skipped %u", + chain_seen, chain_totals.failed, chain_totals.skipped); + check( pushed && chain_dones == 1 && chain_seen == 6 + && chain_saw(0, "fsmain_missing_dir", TASK_DBSCAN_FAILED, -1) + && chain_saw(1, a, TASK_DBSCAN_FAILED, -1) + && chain_saw(2, a, TASK_DBSCAN_FAILED, -1) + && chain_saw(3, "f_gone", TASK_DBSCAN_SKIPPED, -1) + && chain_saw(4, p, TASK_DBSCAN_FAILED, -1) + && chain_saw(5, "f_none", TASK_DBSCAN_FAILED, -1) + && chain_totals.failed == 5 && chain_totals.skipped == 1, + "fs off main: each item stops where it did", detail); + snprintf(detail, sizeof(detail), "%u calls, the last %s", + fs_main_calls, *fs_main_last ? fs_main_last : "none"); + check(fs_main_calls == 0, + "fs off main: no file system call on the main thread", detail); + + manual_content_scan_reset_settings(); + strlcpy(config_get_ptr()->paths.directory_playlist, pl_dir, + sizeof(config_get_ptr()->paths.directory_playlist)); + return; + +fixture: + check(0, "fixture", "could not write the fs off main lane"); + strlcpy(config_get_ptr()->paths.directory_playlist, pl_dir, + sizeof(config_get_ptr()->paths.directory_playlist)); +#else + (void)root; + (void)pl_dir; +#endif +} + +/* The summary the menu's Refresh All Playlists shows. */ +static char menu_all_msg[128]; + +static void menu_all_done_cb(const task_dbscan_chain_result_t *res, + void *user_data) +{ + if (user_data) + chain_ud_ok = false; + snprintf(menu_all_msg, sizeof(menu_all_msg), + msg_hash_to_str(MSG_PLAYLIST_MANAGER_REFRESH_ALL_DONE), + res->ran, res->skipped, res->failed); + chain_totals = *res; + chain_dones++; +} + +/* The one item the menu pushes. */ +static task_dbscan_item_t *menu_all_item(void) +{ + task_dbscan_item_t *item = (task_dbscan_item_t*)calloc(1, + sizeof(*item)); + + if (item) + item->kind = TASK_DBSCAN_ITEM_REFRESH_ALL; + return item; +} + +/* Refresh All Playlists as the menu pushes it, over A refreshable, B + * with its folder gone, C without settings and E with a missing core: + * the English summary, nothing printed, and a second press while it + * runs refused with the English message. */ +static void menu_refresh_all_lane(const char *root, const char *pl_dir, + uint32_t crc_a, uint32_t sz_a, bool threaded) +{ + char pl[512], a[512], gone[512], out[1024], err[1024]; + char p[1024], stale[1024], what[128]; + const char *q = threaded ? "(threaded)" : "(regular)"; + const char *busy; + bool captured, pushed, refused; + + sprintf(pl, "%s/menu_all_pl", root); + sprintf(a, "%s/menu_all_a", root); + sprintf(gone, "%s/menu_all_gone", root); + sprintf(out, "%s/menu_all_out.txt", root); + sprintf(err, "%s/menu_all_err.txt", root); + sprintf(stale, "%s/00_gone.bin", a); + path_mkdir(pl); + path_mkdir(a); + delete_playlists(pl); + sprintf(p, "%s/01_alpha.bin", a); + if (!write_content(p, crc_a, sz_a)) + goto fixture; + sprintf(p, "%s/a_ok.lpl", pl); + if (!refresh_write(p, a, NULL, stale)) + goto fixture; + sprintf(p, "%s/b_gone.lpl", pl); + if (!refresh_write(p, gone, NULL, stale)) + goto fixture; + sprintf(p, "%s/c_none.lpl", pl); + if (!refresh_write(p, NULL, NULL, stale)) + goto fixture; + sprintf(p, "%s/e_core.lpl", pl); + if (!refresh_write(p, a, "No Such Core", stale)) + goto fixture; + strlcpy(config_get_ptr()->paths.directory_playlist, pl, + sizeof(config_get_ptr()->paths.directory_playlist)); + use_queue(threaded); + + chain_seen = 0; + chain_dones = 0; + chain_ud_ok = true; + menu_all_msg[0] = '\0'; + cli_output = false; + captured = capture_begin(out, err); + pushed = task_push_dbscan_chain(menu_all_item(), 1, NULL, + menu_all_done_cb, NULL); + refused = !task_push_dbscan_chain(menu_all_item(), 1, NULL, + menu_all_done_cb, NULL); + chain_wait(false); + if (captured) + capture_end(); + sprintf(what, "menu refresh all %s: the summary", q); + check( pushed && chain_dones == 1 && chain_ud_ok + && !strcmp(menu_all_msg, + "Playlists refreshed: 1, skipped: 1, failed: 1"), + what, menu_all_msg); + busy = msg_hash_to_str(MSG_MANUAL_CONTENT_SCAN_ALREADY_RUNNING); + sprintf(what, "menu refresh all %s: a second press refused", q); + check( refused && chain_dones == 1 + && !strcmp(busy, "A content scan is already running."), + what, busy); + sprintf(what, "menu refresh all %s: nothing printed", q); + check( captured && path_get_size(out) == 0 + && path_get_size(err) == 0, what, err); + + manual_content_scan_reset_settings(); + strlcpy(config_get_ptr()->paths.directory_playlist, pl_dir, + sizeof(config_get_ptr()->paths.directory_playlist)); + return; + +fixture: + check(0, "fixture", "could not write the menu refresh-all lane"); + strlcpy(config_get_ptr()->paths.directory_playlist, pl_dir, + sizeof(config_get_ptr()->paths.directory_playlist)); +} + int main(int argc, char **argv) { const char *root = (argc > 1) ? argv[1] : "/tmp"; @@ -7796,7 +10426,7 @@ int main(int argc, char **argv) fprintf(f, "display_name = \"Scan Test\"\n" "corename = \"ScanTest\"\n" - "supported_extensions = \"bin|cue|gdi|zip\"\n" + "supported_extensions = \"bin|cue|gdi|m3u|zip\"\n" "database = \"Test Alpha|Test Beta|Test Disc|Test Zip\"\n"); fclose(f); } @@ -7945,6 +10575,8 @@ int main(int argc, char **argv) } sprintf(p, "%s/rarcart_libretro.so", core_dir); { FILE *f = fopen(p, "wb"); if (f) { fputs("\177ELF", f); fclose(f); } } + if (!write_lookup_cores(info_dir, core_dir)) + { check(0, "fixture", "could not write core info"); return 1; } #ifdef HAVE_THREADS task_queue_init(true, msgq_push); @@ -8583,6 +11215,7 @@ int main(int argc, char **argv) "empty folder reported as invalid content", !scan_completed ? "timed out" : invalid_content_pushed ? "message pushed" : "no message"); + check_result("and fails the scan", 0, 0, 0, 0, true); } /* A sheet that names itself is still the entry being scanned, so @@ -8747,6 +11380,20 @@ int main(int argc, char **argv) walk_lane(root, pl_dir, db_dir); single_lane(root, pl_dir, crc_a, sz_a); + counts_lane(root, pl_dir, crc_a, sz_a, crc_d, sz_d); + failed_lane(root, pl_dir, db_dir, crc_a, sz_a); + errors_lane(root, pl_dir, crc_a, sz_a); + output_lane(root, pl_dir, db_dir); + reset_lane(root, pl_dir); + overrides_lane(root, pl_dir); + core_lookup_lane(root); + chain_lane(root, pl_dir, crc_a, sz_a, false); + chain_lane(root, pl_dir, crc_a, sz_a, true); + refresh_lane(root, pl_dir, crc_a, sz_a, false); + refresh_lane(root, pl_dir, crc_a, sz_a, true); + fs_main_lane(root, pl_dir); + menu_refresh_all_lane(root, pl_dir, crc_a, sz_a, false); + menu_refresh_all_lane(root, pl_dir, crc_a, sz_a, true); { char reader_db[512]; @@ -8795,6 +11442,9 @@ int main(int argc, char **argv) folder_first_lane(root, pl_dir, false); loose_held_lane(root, pl_dir, true); loose_held_lane(root, pl_dir, false); + refresh_db_name_lane(root); + omit_ref_lane(root, pl_dir, true); + omit_ref_lane(root, pl_dir, false); #ifdef HAVE_RRAR rar_lane(root, pl_dir, true); rar_lane(root, pl_dir, false); diff --git a/samples/tasks/database/main.c b/samples/tasks/database/main.c index 3d6b7b0971bd..e74be51459a7 100644 --- a/samples/tasks/database/main.c +++ b/samples/tasks/database/main.c @@ -239,7 +239,7 @@ int main(int argc, char *argv[]) /* The return value matters: false means no task exists, so nothing * will ever call main_db_cb and the loop below would never end. */ - if (!task_push_dbscan(input_dir, main_db_cb)) + if (!task_push_dbscan(input_dir, main_db_cb, NULL)) { fprintf(stderr, "task_push_dbscan refused to start a scan\n"); goto done; diff --git a/samples/tasks/database/scan_begin_budget_test.c b/samples/tasks/database/scan_begin_budget_test.c index d5408da5d8f4..e348661f919b 100644 --- a/samples/tasks/database/scan_begin_budget_test.c +++ b/samples/tasks/database/scan_begin_budget_test.c @@ -527,7 +527,7 @@ static bool run_scan(const char *content_dir, const char *dat_path, return false; t0 = cpu_features_get_time_usec(); - if (!task_push_manual_content_scan(false, scan_cb)) + if (!task_push_manual_content_scan(false, scan_cb, NULL)) { fprintf(stderr, "task_push_manual_content_scan refused\n"); return false; @@ -628,7 +628,7 @@ static bool run_db_scan(const char *content_dir, const char *db_dir, return false; t0 = cpu_features_get_time_usec(); - if (!task_push_manual_content_scan(false, scan_cb)) + if (!task_push_manual_content_scan(false, scan_cb, NULL)) { fprintf(stderr, "database scan: push refused\n"); return false; @@ -693,7 +693,7 @@ static bool run_scan_threaded(const char *content_dir, const char *dat_path, task_queue_init(true, msgq_push); - if (!task_push_manual_content_scan(false, scan_cb)) + if (!task_push_manual_content_scan(false, scan_cb, NULL)) { fprintf(stderr, "threaded: task_push_manual_content_scan refused\n"); task_queue_deinit(); @@ -742,7 +742,7 @@ static bool run_scan_cancelled(const char *content_dir, const char *dat_path, task_queue_init(false, msgq_push); - if (!task_push_manual_content_scan(false, scan_cb)) + if (!task_push_manual_content_scan(false, scan_cb, NULL)) { fprintf(stderr, "cancel lane: push refused\n"); task_queue_deinit(); diff --git a/tasks/task_database.c b/tasks/task_database.c index d648aa66fb79..075d7d08797a 100644 --- a/tasks/task_database.c +++ b/tasks/task_database.c @@ -30,6 +30,10 @@ #include #include #include +#if !defined(_WIN32) && !defined(ORBIS) \ + && (defined(__unix__) || defined(__APPLE__)) +#include +#endif #ifdef HAVE_THREADS #include #include @@ -139,6 +143,16 @@ static bool scan_results_add(scan_results_t *sr, return true; } +static scan_result_t *scan_results_find(scan_results_t *sr, + const char *entry_path) +{ + size_t i; + for (i = 0; i < sr->count; i++) + if (string_is_equal(sr->results[i].entry_path, entry_path)) + return &sr->results[i]; + return NULL; +} + static void scan_results_free(scan_results_t *sr) { size_t i; @@ -592,6 +606,14 @@ typedef struct manual_scan_handle * this is how a task_push_dbscan caller learns the scan finished; * see cb_task_manual_content_scan. */ retro_task_callback_t user_cb; + void *user_data; + /* The callback's task_data. */ + task_dbscan_result_t result; + /* What a scan that could not start told the user. */ + const char *fail_msg; + /* What the --scan lines call the scan instead of its content + * folder: a refreshed playlist's name. */ + char *cli_name; } manual_scan_handle_t; enum scan_verdict @@ -638,16 +660,43 @@ static const char *database_info_get_current_element_name( return handle->elems[ptr].data; } #endif -static void task_database_scan_console_output(const char *label, const char *db_name, bool add) +enum scan_console_mark { + SCAN_CONSOLE_ADDED = 0, + SCAN_CONSOLE_DUPLICATE, + SCAN_CONSOLE_UNMATCHED, + SCAN_CONSOLE_ERROR +}; + +/* Whether @stream is a terminal. Where that cannot be told, say yes. */ +static bool task_database_scan_console_is_tty(FILE *stream) +{ +#if defined(_WIN32) && !defined(_XBOX) && !defined(__WINRT__) + DWORD mode; + return GetConsoleMode(GetStdHandle((stream == stderr) + ? STD_ERROR_HANDLE : STD_OUTPUT_HANDLE), &mode) != 0; +#elif (defined(__unix__) || defined(__APPLE__)) && !defined(ORBIS) + /* The PS4 toolchain defines __unix__ but has no isatty(). */ + return isatty(fileno(stream)) != 0; +#else + return true; +#endif +} + +static void task_database_scan_console_output(FILE *stream, + const char *label, const char *db_name, enum scan_console_mark mark) +{ + static const char *const prefixes[] = { "++", "==", "??", "!!" }; char string[32]; - const char *prefix = (add) ? "++" : (db_name) ? "==" : "??"; + const char *prefix = prefixes[mark]; const char *no_color = getenv("NO_COLOR"); - bool color = (no_color && no_color[0] != '0') ? false : true; + bool color = (no_color && no_color[0] != '0') + ? false : task_database_scan_console_is_tty(stream); - /* Colorize prefix (add = green, dupe = yellow, not found = red) */ + /* Colorize prefix (add = green, dupe = yellow, not found or error = red) */ #ifdef _WIN32 - HANDLE con = GetStdHandle(STD_OUTPUT_HANDLE); + HANDLE con = GetStdHandle((stream == stderr) + ? STD_ERROR_HANDLE : STD_OUTPUT_HANDLE); if (color && con != INVALID_HANDLE_VALUE) { unsigned red = FOREGROUND_RED; @@ -657,9 +706,11 @@ static void task_database_scan_console_output(const char *label, const char *db_ size_t _len = strlcpy_lit(string, " ", sizeof(string)); _len += strlcpy(string + _len, prefix, sizeof(string) - _len); _len += strlcpy_lit(string + _len, " ", sizeof(string) - _len); - SetConsoleTextAttribute(con, (add) ? green : (db_name) ? yellow : red); + SetConsoleTextAttribute(con, (mark == SCAN_CONSOLE_ADDED) ? green + : (mark == SCAN_CONSOLE_DUPLICATE) ? yellow : red); WriteConsole(con, string, _len, NULL, NULL); SetConsoleTextAttribute(con, reset); + string[0] = '\0'; } #else if (color) @@ -669,15 +720,16 @@ static void task_database_scan_console_output(const char *label, const char *db_ const char *yellow = "\x1B[33m"; const char *reset = "\x1B[0m"; size_t _len = 0; - if (add) + if (mark == SCAN_CONSOLE_ADDED) _len += strlcpy(string + _len, green, sizeof(string) - _len); else - _len += strlcpy(string + _len, (db_name) ? yellow : red, sizeof(string) - _len); + _len += strlcpy(string + _len, + (mark == SCAN_CONSOLE_DUPLICATE) ? yellow : red, + sizeof(string) - _len); _len += strlcpy_lit(string + _len, " ", sizeof(string) - _len); _len += strlcpy(string + _len, prefix, sizeof(string) - _len); _len += strlcpy_lit(string + _len, " ", sizeof(string) - _len); strlcpy(string + _len, reset, sizeof(string) - _len); - fputs(string, stdout); } #endif else @@ -685,13 +737,14 @@ static void task_database_scan_console_output(const char *label, const char *db_ size_t _len = strlcpy_lit(string, " ", sizeof(string)); _len += strlcpy(string + _len, prefix, sizeof(string) - _len); strlcpy_lit(string + _len, " ", sizeof(string) - _len); - fputs(string, stdout); } + /* One call, so another thread's log line cannot split it. The + * Win32 colour prefix is written apart, before it. */ if (!db_name) - printf("\"%s\"\n", label); + fprintf(stream, "%s\"%s\"\n", string, label); else - printf("\"%s / %s\"\n", db_name, label); + fprintf(stream, "%s\"%s / %s\"\n", string, db_name, label); } #ifdef HAVE_LIBRETRODB static enum scan_verdict task_database_iterate_start(retro_task_t *task, @@ -908,7 +961,7 @@ static void remove_disc_indicators(char *title, size_t len) static void task_database_iterate_m3u( manual_scan_handle_t *_db, - const char *m3u_path) + const char *m3u_path, bool counted) { size_t i, j; bool found_match = false; @@ -916,6 +969,7 @@ static void task_database_iterate_m3u( char first_matched_crc[128]; char collapsed_title[NAME_MAX_LENGTH]; rm3u_t *m3u = NULL; + scan_result_t *own; first_matched_db[0] = '\0'; first_matched_crc[0] = '\0'; @@ -925,6 +979,8 @@ static void task_database_iterate_m3u( if (!(m3u = rm3u_load_filestream(m3u_path))) { RARCH_ERR("[Scanner] Failed to open M3U file: \"%s\".\n", m3u_path); + if (!counted && !scan_results_find(&_db->scan_results, m3u_path)) + _db->result.unmatched++; return; } @@ -1013,9 +1069,19 @@ static void task_database_iterate_m3u( rm3u_free(m3u); + own = scan_results_find(&_db->scan_results, m3u_path); + /* If we found at least one match, add M3U entry */ if (found_match) { + /* The content pass's entry for the m3u itself would only be + * replaced by this one in the same playlist. */ + if (own && ( *_db->task_config->playlist_file + || string_is_equal(own->db_name, first_matched_db))) + { + free(own->entry_path); + own->entry_path = NULL; + } if (!scan_results_add(&_db->scan_results, m3u_path, collapsed_title, first_matched_crc, first_matched_db, NULL)) { @@ -1027,6 +1093,8 @@ static void task_database_iterate_m3u( collapsed_title, first_matched_db); } } + else if (!own && !counted) + _db->result.unmatched++; /* Compact scan_results to remove NULL entries */ { @@ -2211,7 +2279,8 @@ static enum scan_verdict database_info_list_iterate_end_no_match( /* Reached end of database list, * CRC match probably didn't succeed. */ if (_db->task_config->cli_scan_output) - task_database_scan_console_output(path, NULL, false); + task_database_scan_console_output(stdout, path, NULL, + SCAN_CONSOLE_UNMATCHED); /* With its directory read, an archive's held hits were its * expansion. It still expands where a member stores no CRC, and @@ -2285,6 +2354,7 @@ static bool task_database_result_add(manual_scan_handle_t *_db, : (char*)malloc(db_crc_len); char* entry_path_str = (char*)malloc(str_len); char *hash = NULL; + const char *db_ref = db_playlist_base_str; const char *db_path = database_info_get_current_name(db_state); const char *entry_path = @@ -2368,11 +2438,14 @@ static bool task_database_result_add(manual_scan_handle_t *_db, && (hash = strchr(entry_path_str, '#'))) *hash = '\0'; + /* Without a fixed playlist, db_name picks the playlist. */ + if (_db->task_config->omit_db_reference && *_db->task_config->playlist_file) + db_ref = path_basename_nocompression(_db->task_config->playlist_file); + /* Accumulate result instead of immediately updating playlist */ - if (!scan_results_add(&_db->scan_results, entry_path_str, entry_lbl, db_crc, - _db->task_config->omit_db_reference ? _db->task_config->dat_file_path : db_playlist_base_str, - archive_name)) + if (!scan_results_add(&_db->scan_results, entry_path_str, entry_lbl, db_crc, + db_ref, archive_name)) RARCH_ERR("[Scanner] Failed to add result for: \"%s\".\n", entry_lbl); if (db_crc != db_crc_buf) @@ -4084,17 +4157,27 @@ static bool manual_scan_end_flush_tick( * playlist for the same answer. */ playlist_push_unchecked(manual_scan->flush_playlist, &entry); manual_scan->flush_added++; + /* A present m3u is replaced, not added. */ + if (entry_present) + manual_scan->result.duplicates++; + else + manual_scan->result.added++; RARCH_LOG("[Scanner] Add \"%s / %s\".\n", db_name_noext, result->entry_label); if (manual_scan->task_config->cli_scan_output) - task_database_scan_console_output(result->entry_label, - db_name_noext, true); + task_database_scan_console_output(stdout, + result->entry_label, db_name_noext, SCAN_CONSOLE_ADDED); } /* Entry already exists - output duplicate indicator for CLI scans */ - else if (manual_scan->flush_playlist && manual_scan->task_config->cli_scan_output) - task_database_scan_console_output(result->entry_label, - db_name_noext, false); + else if (manual_scan->flush_playlist) + { + manual_scan->result.duplicates++; + if (manual_scan->task_config->cli_scan_output) + task_database_scan_console_output(stdout, + result->entry_label, db_name_noext, + SCAN_CONSOLE_DUPLICATE); + } manual_scan->flush_pos++; } @@ -4128,10 +4211,13 @@ static bool manual_scan_end_flush_tick( } #ifdef HAVE_LIBRETRODB -bool task_push_dbscan(const char *fullpath, retro_task_callback_t cb) +bool task_push_dbscan(const char *fullpath, retro_task_callback_t cb, + void *user_data) { + if (task_dbscan_chain_refuse()) + return false; manual_content_scan_set_menu_content_dir(fullpath); - return task_push_manual_content_scan(false, cb); + return task_push_manual_content_scan(false, cb, user_data); } #endif @@ -4239,6 +4325,9 @@ static void free_manual_content_scan_handle(manual_scan_handle_t *manual_scan) free(manual_scan->playlist_directory); manual_scan->playlist_directory = NULL; + free(manual_scan->cli_name); + manual_scan->cli_name = NULL; + #ifdef HAVE_LIBRETRODB if (1) { @@ -4393,7 +4482,12 @@ static void cb_task_manual_content_scan( * documented as menu-only and the sample in samples/tasks/database * hangs on exactly this. */ if (manual_scan && manual_scan->user_cb) - manual_scan->user_cb(task, task_data, user_data, err); + { + if (err) + manual_scan->result.failed = true; + manual_scan->user_cb(task, &manual_scan->result, + manual_scan->user_data, err); + } #if defined(HAVE_MENU) /* When creating playlists, the playlist tabs of @@ -4579,9 +4673,14 @@ static bool manual_scan_begin_setup(retro_task_t *task, } } - RARCH_LOG("[Scanner] %s\"%s\"...\n", msg_hash_to_str(MSG_MANUAL_CONTENT_SCAN_START), manual_scan->content_database_path); + /* A missing or unset folder holds no database, as an empty + * one does: the end step reports it. */ + if (!dbstate->list && !(dbstate->list = string_list_new())) + return true; + + RARCH_LOG("[Scanner] %s\"%s\"...\n", msg_hash_to_str(MSG_MANUAL_CONTENT_SCAN_START), manual_scan->task_config->content_dir); if (manual_scan->task_config->cli_scan_output) - printf("%s\"%s\"...\n", msg_hash_to_str(MSG_MANUAL_CONTENT_SCAN_START), manual_scan->content_database_path); + printf("%s\"%s\"...\n", msg_hash_to_str(MSG_MANUAL_CONTENT_SCAN_START), manual_scan->task_config->content_dir); } } else @@ -4605,6 +4704,7 @@ static bool manual_scan_begin_setup(retro_task_t *task, const char *_msg = msg_hash_to_str(MSG_MANUAL_CONTENT_SCAN_INVALID_CONTENT); runloop_msg_queue_push(_msg, strlen(_msg), 1, 100, true, NULL,\ MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_INFO); + manual_scan->fail_msg = _msg; return true; } } @@ -4646,6 +4746,7 @@ static bool manual_scan_begin_dir_list(retro_task_t *task, const char *_msg = msg_hash_to_str(MSG_MANUAL_CONTENT_SCAN_INVALID_CONTENT); runloop_msg_queue_push(_msg, strlen(_msg), 1, 100, true, NULL,\ MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_INFO); + manual_scan->fail_msg = _msg; } return true; } @@ -4664,6 +4765,7 @@ static bool manual_scan_begin_dir_list(retro_task_t *task, const char *_msg = msg_hash_to_str(MSG_MANUAL_CONTENT_SCAN_INVALID_CONTENT); runloop_msg_queue_push(_msg, strlen(_msg), 1, 100, true, NULL, MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_INFO); + manual_scan->fail_msg = _msg; return true; } @@ -4687,6 +4789,7 @@ static bool manual_scan_begin_dir_list(retro_task_t *task, const char *_msg = msg_hash_to_str(MSG_MANUAL_CONTENT_SCAN_INVALID_CONTENT); runloop_msg_queue_push(_msg, strlen(_msg), 1, 100, true, NULL,\ MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_INFO); + manual_scan->fail_msg = _msg; return true; } dir_list_sort(manual_scan->content_list, true); @@ -4828,6 +4931,7 @@ static bool manual_scan_begin_dat_load(retro_task_t *task, const char *_msg = msg_hash_to_str(MSG_MANUAL_CONTENT_SCAN_DAT_FILE_LOAD_ERROR); runloop_msg_queue_push(_msg, strlen(_msg), 1, 100, true, NULL, MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_INFO); + manual_scan->fail_msg = _msg; } return true; } @@ -5142,6 +5246,9 @@ static enum scan_step task_manual_scan_iterate_step(retro_task_t *task, manual_scan->status = MANUAL_SCAN_ITERATE_CONTENT; else { + /* An m3u is counted in the m3u pass. */ + if (!rm3u_is_m3u_filestream(content_path)) + manual_scan->result.unmatched++; manual_scan->status = DATABASE_SCAN_ITERATE_NEXT; } break; @@ -5154,6 +5261,18 @@ static enum scan_step task_manual_scan_iterate_step(retro_task_t *task, * same failing step every tick. Give up on this * file and go to the next one. */ RARCH_ERR("[Scanner] Scanning of content unexpectedly failed for \"%s\"\n", content_path); + /* Counted here, so not again in the m3u pass. */ + if ( manual_scan->m3u_list + && manual_scan->m3u_list->size + && string_is_equal(manual_scan->m3u_list->elems[ + manual_scan->m3u_list->size - 1].data, + content_path)) + manual_scan->m3u_list->elems[ + manual_scan->m3u_list->size - 1].attr.i = 1; + manual_scan->result.errors++; + if (manual_scan->task_config->cli_scan_output) + task_database_scan_console_output(stderr, + content_path, NULL, SCAN_CONSOLE_ERROR); #ifdef SCAN_HAVE_READERS if (dbstate->crc_pass != DB_CRC_PASS_FILE) scan_readers_whole_ahead(manual_scan, 0); @@ -5249,6 +5368,11 @@ static enum scan_step task_manual_scan_iterate_step(retro_task_t *task, *manual_scan->task_config->database_name ? manual_scan->task_config->database_name : manual_scan->task_config->dat_file_path; + /* Skip Database References, as for a match */ + if ( manual_scan->task_config->omit_db_reference + && *manual_scan->task_config->playlist_file) + db_name = path_basename_nocompression( + manual_scan->task_config->playlist_file); label[0] = '\0'; if (!manual_content_scan_get_playlist_content_label( content_path, manual_scan->dat_file, @@ -5256,6 +5380,8 @@ static enum scan_step task_manual_scan_iterate_step(retro_task_t *task, label, sizeof(label))) { label[0] = '\0'; + if (!rm3u_is_m3u_filestream(content_path)) + manual_scan->result.unmatched++; #ifdef DEBUG RARCH_DBG("[Scanner] Rejecting item: %s\n",content_path); #endif @@ -5368,6 +5494,13 @@ static bool task_manual_scan_iterate_tick(retro_task_t *task, return finished; } +static const char *manual_scan_cli_name( + const manual_scan_handle_t *manual_scan) +{ + return manual_scan->cli_name + ? manual_scan->cli_name : manual_scan->task_config->content_dir; +} + static void task_manual_content_scan_handler(retro_task_t *task) { uint8_t flg; @@ -5416,7 +5549,18 @@ static void task_manual_content_scan_handler(retro_task_t *task) case MANUAL_SCAN_BEGIN_DAT_LOAD: case MANUAL_SCAN_BEGIN_PLAYLIST: if (task_manual_scan_begin_tick(task, manual_scan)) + { + manual_scan->result.failed = true; + if (manual_scan->task_config->cli_scan_output) + { + fprintf(stderr, "%s: %s\n", + manual_scan_cli_name(manual_scan), + manual_scan->fail_msg ? manual_scan->fail_msg + : msg_hash_to_str(MSG_MANUAL_CONTENT_SCAN_INVALID_CONFIG)); + fflush(stdout); + } goto task_finished; + } break; case MANUAL_SCAN_ITERATE_M3U: { @@ -5445,7 +5589,9 @@ static void task_manual_content_scan_handler(retro_task_t *task) task_set_progress(task, (manual_scan->m3u_index * 100) / manual_scan->m3u_list->size); - task_database_iterate_m3u(manual_scan, m3u_path); + task_database_iterate_m3u(manual_scan, m3u_path, + manual_scan->m3u_list->elems[ + manual_scan->m3u_index].attr.i != 0); } /* Increment M3U file index */ @@ -5456,7 +5602,8 @@ static void task_manual_content_scan_handler(retro_task_t *task) break; case MANUAL_SCAN_END: { - const char *msg = NULL; + const char *msg = NULL; + bool no_database = false; /* Batch update all playlists with accumulated results, * spread across gathers under the shared window. */ @@ -5478,6 +5625,7 @@ static void task_manual_content_scan_handler(retro_task_t *task) { msg = msg_hash_to_str(MSG_SCANNING_NO_DATABASE); task_set_error(task, strdup(msg)); + no_database = true; } else { @@ -5494,7 +5642,24 @@ static void task_manual_content_scan_handler(retro_task_t *task) task_set_progress(task, 100); RARCH_LOG("[Scanner] %s\n", msg); if (manual_scan->task_config->cli_scan_output) - printf("%s\n", msg); + { + if (no_database) + fprintf(stderr, "%s: %s\n", + manual_scan_cli_name(manual_scan), msg); + else + { + printf("%s\n", msg); + printf("%s: %u added, %u duplicates, %u unmatched, %u errors\n", + manual_scan_cli_name(manual_scan), + manual_scan->result.added, + manual_scan->result.duplicates, + manual_scan->result.unmatched, + manual_scan->result.errors); + } + /* With --menu the process goes on: hand each scan's + * lines to a pipe or file as it ends. */ + fflush(stdout); + } RARCH_DBG("[Scanner] Scan settings were:\n"); RARCH_DBG("[Scanner] Content dir: \"%s\"\n",manual_scan->task_config->content_dir); @@ -5543,15 +5708,21 @@ static bool task_manual_content_scan_finder(retro_task_t *task, void *user_data) return false; if (task->handler != task_manual_content_scan_handler) return false; + /* Still findable while its callback runs, which may push the + * next scan. */ + if (task_get_flags(task) & RETRO_TASK_FLG_FINISHED) + return false; if (!(manual_scan = (manual_scan_handle_t*)task->state)) return false; return string_is_equal( (const char*)user_data, manual_scan->playlist_config.path); } -bool task_push_manual_content_scan( - bool do_menu_refresh, - retro_task_callback_t user_cb) +/* @cli_name: what the --scan lines call the scan, NULL for its + * content folder. */ +static bool task_manual_content_scan_push( + bool do_menu_refresh, const char *cli_name, + retro_task_callback_t user_cb, void *user_data) { size_t _len; task_finder_data_t find_data; @@ -5585,6 +5756,9 @@ bool task_push_manual_content_scan( if (!manual_scan->m3u_list) goto error; + if (cli_name && !(manual_scan->cli_name = strdup(cli_name))) + goto error; + manual_scan->playlist_config.capacity = COLLECTION_SIZE; manual_scan->playlist_config.old_format = settings->bools.playlist_use_old_format; manual_scan->playlist_config.compress = settings->bools.playlist_compression; @@ -5675,6 +5849,7 @@ bool task_push_manual_content_scan( #endif manual_scan->user_cb = user_cb; + manual_scan->user_data = user_data; task->callback = cb_task_manual_content_scan; task->cleanup = task_manual_content_scan_free; task->flags |= RETRO_TASK_FLG_ALTERNATIVE_LOOK; @@ -5692,3 +5867,1162 @@ bool task_push_manual_content_scan( return false; } + +bool task_push_manual_content_scan( + bool do_menu_refresh, + retro_task_callback_t user_cb, void *user_data) +{ + return task_manual_content_scan_push(do_menu_refresh, NULL, user_cb, + user_data); +} + +/* --- The scan chain ----------------------------------------------- */ + +enum dbscan_prep_kind +{ + DBSCAN_PREP_DBS = 0, /* the database folder */ + DBSCAN_PREP_LIST, /* the playlist folder */ + DBSCAN_PREP_LOAD, /* one playlist */ + DBSCAN_PREP_PATH /* a --scan path */ +}; + +enum dbscan_load_status +{ + DBSCAN_LOAD_OK = 0, + DBSCAN_LOAD_NO_PLAYLIST, + DBSCAN_LOAD_AMBIGUOUS, + DBSCAN_LOAD_NO_SETTINGS, + DBSCAN_LOAD_FOLDER_MISSING, + DBSCAN_LOAD_OUTSIDE, /* a --scan-refresh path not in the folder */ + DBSCAN_LOAD_NO_MEMORY +}; + +/* A prep task's state: the handler fills it, and the callback takes + * what it needs on the main thread once the task finished. */ +typedef struct +{ + playlist_config_t config; /* LOAD: the playlist */ + char *dir; /* DBS, LIST; a --scan-refresh's LOAD */ + char *name; /* LOAD: the name its lines give */ + /* PATH, and a --scan-refresh's LOAD: the item as typed; that LOAD's + * playlist folder as configured. Copies, as are the overrides' + * core and DAT: the worker reads nothing of the chain. */ + char *arg; + char *pl_dir; + char *ov_core; + char *ov_dat; + char *path; /* PATH: resolved, NULL when not there */ + char *detail; /* LOAD: OUTSIDE's folder, for its line */ + manual_content_scan_fs_t *fs; /* PATH, LOAD: what the file system said */ + dir_list_iter_t *iter; + struct string_list *list; + playlist_parse_t *parse; + playlist_t *playlist; + enum dbscan_prep_kind kind; + enum dbscan_load_status load; + unsigned matches; /* LOAD: files in another case, if ambiguous */ + bool include_hidden; + bool begun; + bool failed; /* LIST, PATH: out of memory */ +} dbscan_prep_t; + +typedef struct +{ + task_dbscan_item_t *items; + struct string_list *rdb_list; + struct string_list *playlists; /* the running refresh-all's */ + char *name; /* the running scan's */ + task_dbscan_item_cb_t item_cb; + task_dbscan_chain_cb_t done_cb; + void *user_data; + size_t count; + size_t next; + size_t all_next; + task_dbscan_chain_result_t totals; + /* the Manual Scan settings the chain found, put back when it ends */ + manual_content_scan_saved_t *saved; + bool all_running; + bool all_counted; /* a playlist of the refresh-all had settings */ + bool cli_output; +} dbscan_chain_t; + +/* Main thread only, as every step of the chain. */ +static dbscan_chain_t *dbscan_chain_running = NULL; + +static void dbscan_chain_free_items(task_dbscan_item_t *items, + size_t count) +{ + size_t i; + + if (!items) + return; + for (i = 0; i < count; i++) + { + free(items[i].arg); + manual_content_scan_overrides_free(items[i].overrides); + } + free(items); +} + +/* Ends the chain: what it found in the Manual Scan settings goes back, + * and it is no longer running when @done_cb hears the totals, so that + * callback may start another. */ +static void dbscan_chain_finish(dbscan_chain_t *chain) +{ + manual_content_scan_restore_settings(chain->saved); + chain->saved = NULL; + dbscan_chain_running = NULL; + if (chain->done_cb) + chain->done_cb(&chain->totals, chain->user_data); + dbscan_chain_free_items(chain->items, chain->count); + if (chain->rdb_list) + string_list_free(chain->rdb_list); + if (chain->playlists) + string_list_free(chain->playlists); + free(chain->name); + free(chain); +} + +static void dbscan_chain_fold(dbscan_chain_t *chain, const char *name, + enum task_dbscan_outcome outcome, const task_dbscan_result_t *res) +{ + switch (outcome) + { + case TASK_DBSCAN_RAN: + chain->totals.ran++; + if (res && res->added) + chain->totals.added++; + break; + case TASK_DBSCAN_FAILED: + chain->totals.failed++; + break; + case TASK_DBSCAN_SKIPPED: + chain->totals.skipped++; + break; + } + if (chain->item_cb) + chain->item_cb(name, outcome, res, chain->user_data); +} + +/* @in with '~' expanded, made absolute if relative; NULL on + * allocation failure. Free the result. */ +static char *dbscan_chain_expand(const char *in) +{ + char *s = (char*)malloc(PATH_MAX_LENGTH); + + if (!s) + return NULL; + fill_pathname_expand_special(s, in, PATH_MAX_LENGTH); + /* "" would resolve to the working directory. Consoles resolve + * nothing, so a relative path stays as it is there. */ +#ifndef RARCH_CONSOLE + if ( *s && !path_is_absolute(s) + && !path_resolve_realpath(s, PATH_MAX_LENGTH, false)) + *s = '\0'; +#endif + return s; +} + +/* Resolves a --scan path and reads what its scan will ask of the + * file system: on a threaded queue's worker, else at once on the main + * thread, as a stat is one call. */ +static void dbscan_path_handler(retro_task_t *task) +{ + dbscan_prep_t *prep = (dbscan_prep_t*)task->state; + + if (!(task_get_flags(task) & RETRO_TASK_FLG_CANCELLED)) + { + char *s = dbscan_chain_expand(prep->arg); + + if (!s) + prep->failed = true; + else if (!*s) + free(s); + else if (!(prep->fs = manual_content_scan_fs_read(s, NULL, + prep->ov_core, prep->ov_dat))) + { + prep->failed = true; + free(s); + } + /* The stat the content folder's setter takes */ + else if (!(prep->fs->content_stat & RETRO_VFS_STAT_IS_VALID)) + free(s); + else + prep->path = s; + } + task_set_flags(task, RETRO_TASK_FLG_FINISHED, true); +} + +/* @name: the item, for the one error the overrides cannot name. */ +static void dbscan_chain_override_error(const dbscan_chain_t *chain, + const char *name, const manual_content_scan_overrides_t *ov, + enum manual_content_scan_override_status st, unsigned count) +{ + if (!chain->cli_output) + return; + switch (st) + { + case MANUAL_CONTENT_SCAN_OVR_NO_SUCH_CORE: + fprintf(stderr, "%s: no such core\n", ov->core); + break; + case MANUAL_CONTENT_SCAN_OVR_AMBIGUOUS_CORE: + fprintf(stderr, "%s: %u cores with this name, give the path\n", + ov->core, count); + break; + case MANUAL_CONTENT_SCAN_OVR_NO_SUCH_DATABASE: + fprintf(stderr, "%s: no such database\n", ov->database); + break; + case MANUAL_CONTENT_SCAN_OVR_NOT_A_DAT: + fprintf(stderr, "%s: not a DAT file\n", ov->dat); + break; + case MANUAL_CONTENT_SCAN_OVR_DAT_TOO_LARGE: + fprintf(stderr, "%s: DAT file too large\n", ov->dat); + break; + case MANUAL_CONTENT_SCAN_OVR_DAT_WITHOUT_MATCH: + fprintf(stderr, + "%s: --scan-dat cannot be used with --scan-match=none\n", + name); + break; + case MANUAL_CONTENT_SCAN_OVR_OK: + break; + } +} + +/* A refresh that does not run: logged, and its line with --scan. + * @detail, when not NULL, follows in brackets. */ +static void dbscan_chain_refresh_error(const dbscan_chain_t *chain, + const char *name, const char *reason, const char *detail) +{ + if (detail) + { + RARCH_ERR("[Playlist Refresh] %s: %s (%s).\n", name, reason, detail); + if (chain->cli_output) + fprintf(stderr, "%s: %s (%s)\n", name, reason, detail); + } + else + { + RARCH_ERR("[Playlist Refresh] %s: %s.\n", name, reason); + if (chain->cli_output) + fprintf(stderr, "%s: %s\n", name, reason); + } +} + +static void dbscan_chain_next(dbscan_chain_t *chain); + +static void dbscan_chain_scan_cb(retro_task_t *task, void *task_data, + void *user_data, const char *err) +{ + dbscan_chain_t *chain = (dbscan_chain_t*)user_data; + const task_dbscan_result_t *res = (const task_dbscan_result_t*)task_data; + char *name = chain->name; + + chain->name = NULL; + dbscan_chain_fold(chain, name, + res->failed ? TASK_DBSCAN_FAILED : TASK_DBSCAN_RAN, res); + free(name); + /* Cancelled, as on quit: start no more. */ + if (task_get_flags(task) & RETRO_TASK_FLG_CANCELLED) + { + chain->totals.cancelled = true; + dbscan_chain_finish(chain); + } + else + dbscan_chain_next(chain); +} + +/* --scan-overwrite, --scan-validate and --scan-omit-db-ref act on one + * playlist only: warns of those in @flags when the applied settings + * give none. */ +static void dbscan_chain_single_warn(const dbscan_chain_t *chain, + const char *name, unsigned flags) +{ + if (!chain->cli_output || manual_content_scan_is_single_playlist()) + return; + if (flags & MANUAL_CONTENT_SCAN_OVR_OVERWRITE) + fprintf(stderr, "%s: --scan-overwrite needs one playlist, ignored\n", + name); + if (flags & MANUAL_CONTENT_SCAN_OVR_VALIDATE) + fprintf(stderr, "%s: --scan-validate needs one playlist, ignored\n", + name); + if (flags & MANUAL_CONTENT_SCAN_OVR_OMIT_DB_REF) + fprintf(stderr, "%s: --scan-omit-db-ref needs one playlist, ignored\n", + name); +} + +/* True once the item's scan is pushed, @prep's path and what the file + * system said of it in hand; otherwise it is folded as failed. */ +static bool dbscan_chain_push_scan(dbscan_chain_t *chain, + const task_dbscan_item_t *item, dbscan_prep_t *prep) +{ + enum manual_content_scan_override_status st; + unsigned count = 0; + char *path = prep->path; + + if (!path) + { + if (prep->failed) + fprintf(stderr, "out of memory\n"); + else + fprintf(stderr, "%s: not found\n", item->arg); + dbscan_chain_fold(chain, item->arg, TASK_DBSCAN_FAILED, NULL); + return false; + } + prep->path = NULL; + manual_content_scan_reset_settings(); + manual_content_scan_set_menu_content_dir_fs(path, prep->fs); + if ((st = manual_content_scan_apply_overrides_fs(item->overrides, + chain->rdb_list, &count, prep->fs)) != MANUAL_CONTENT_SCAN_OVR_OK) + dbscan_chain_override_error(chain, item->arg, item->overrides, st, + count); + else + { + unsigned flags = item->overrides ? item->overrides->flags : 0; + + if ( chain->cli_output + && (flags & MANUAL_CONTENT_SCAN_OVR_OVERWRITE) + && !(prep->fs->content_stat & RETRO_VFS_STAT_IS_DIRECTORY)) + { + fprintf(stderr, "%s: --scan-overwrite ignored for a single file\n", + path); + flags &= ~(unsigned)MANUAL_CONTENT_SCAN_OVR_OVERWRITE; + } + dbscan_chain_single_warn(chain, path, flags); + if (task_push_manual_content_scan(false, dbscan_chain_scan_cb, chain)) + { + chain->name = path; + return true; + } + if (chain->cli_output) + fprintf(stderr, "%s: the scan could not start\n", path); + } + dbscan_chain_fold(chain, path, TASK_DBSCAN_FAILED, NULL); + free(path); + return false; +} + +/* Sets the loaded playlist's saved settings, @item's overrides over + * them, and pushes its scan; otherwise folds it as failed. */ +static bool dbscan_chain_push_refreshed(dbscan_chain_t *chain, + const task_dbscan_item_t *item, dbscan_prep_t *prep) +{ + enum manual_content_scan_override_status st; + unsigned count = 0; + const char *reason = NULL; + const char *detail = NULL; + playlist_t *pl = prep->playlist; + + manual_content_scan_reset_settings(); + switch (manual_content_scan_set_menu_from_playlist_list_fs(pl, + chain->rdb_list, prep->fs)) + { + case MANUAL_CONTENT_SCAN_PLAYLIST_REFRESH_OK: + break; + case MANUAL_CONTENT_SCAN_PLAYLIST_REFRESH_INVALID_CONTENT_DIR: + reason = "invalid content folder"; + detail = playlist_get_scan_content_dir(pl); + break; + case MANUAL_CONTENT_SCAN_PLAYLIST_REFRESH_INVALID_SYSTEM_NAME: + reason = "invalid playlist name"; + break; + case MANUAL_CONTENT_SCAN_PLAYLIST_REFRESH_INVALID_CORE: + reason = "saved core not found"; + detail = playlist_get_default_core_name(pl); + break; + case MANUAL_CONTENT_SCAN_PLAYLIST_REFRESH_INVALID_DAT_FILE: + reason = "not a DAT file"; + detail = playlist_get_scan_dat_file_path(pl); + break; + case MANUAL_CONTENT_SCAN_PLAYLIST_REFRESH_DAT_FILE_TOO_LARGE: + reason = "DAT file too large"; + detail = playlist_get_scan_dat_file_path(pl); + break; + case MANUAL_CONTENT_SCAN_PLAYLIST_REFRESH_MISSING_CONFIG: + default: + reason = "no saved scan settings"; + break; + } + if (reason) + dbscan_chain_refresh_error(chain, prep->name, reason, detail); + playlist_free(pl); + prep->playlist = NULL; + if (!reason) + { + if ((st = manual_content_scan_apply_overrides_fs(item->overrides, + chain->rdb_list, &count, prep->fs)) + != MANUAL_CONTENT_SCAN_OVR_OK) + dbscan_chain_override_error(chain, prep->name, item->overrides, + st, count); + else + { + dbscan_chain_single_warn(chain, prep->name, + item->overrides ? item->overrides->flags : 0); + if (task_manual_content_scan_push(false, prep->name, + dbscan_chain_scan_cb, chain)) + { + chain->name = prep->name; + prep->name = NULL; + return true; + } + dbscan_chain_refresh_error(chain, prep->name, + "the scan could not start", NULL); + } + } + dbscan_chain_fold(chain, prep->name, TASK_DBSCAN_FAILED, NULL); + return false; +} + +/* Drops what Manage Playlists does not list: the histories and the + * favorites. */ +static void dbscan_prep_filter(struct string_list *list) +{ + size_t i; + size_t n = 0; + + for (i = 0; i < list->size; i++) + { + const char *path = list->elems[i].data; + + if ( string_ends_with_size(path, "_history.lpl", strlen(path), + STRLEN_CONST("_history.lpl")) + || string_is_equal(path_basename_nocompression(path), + FILE_PATH_CONTENT_FAVORITES)) + { + free(list->elems[i].data); + free(list->elems[i].userdata); + } + else + list->elems[n++] = list->elems[i]; + } + list->size = n; +} + +/* Lists the database or playlist folder: to the end on a threaded + * queue's worker, else under the shared window. */ +static void dbscan_prep_handler(retro_task_t *task) +{ + dbscan_prep_t *prep = (dbscan_prep_t*)task->state; + int r = 1; + + if (task_get_flags(task) & RETRO_TASK_FLG_CANCELLED) + { + task_set_flags(task, RETRO_TASK_FLG_FINISHED, true); + return; + } + if (!prep->begun) + { + prep->begun = true; + if (prep->dir && (prep->list = string_list_new())) + prep->iter = dir_list_iter_new(prep->dir, + prep->kind == DBSCAN_PREP_LIST ? "lpl" : "rdb", false, + prep->include_hidden, false, false, prep->list); + } + if (prep->iter) + { + if (task_queue_is_threaded()) + r = dir_list_iter_step(prep->iter, NULL, NULL); + else + { + nbio_budget_t b; + task_nbio_slice_open(&b); + r = dir_list_iter_step(prep->iter, + manual_scan_walk_within_budget, &b); + task_nbio_slice_close(&b); + if (!r) + return; + } + } + prep->failed = r < 0 || (prep->dir && !prep->list); + /* No listing, as dir_list_new() gives none. */ + if ((!prep->iter || r < 0) && prep->list) + { + string_list_free(prep->list); + prep->list = NULL; + } + /* In the order Manage Playlists shows. */ + if (prep->kind == DBSCAN_PREP_LIST && prep->list) + { + dbscan_prep_filter(prep->list); + dir_list_sort(prep->list, true); + } + task_set_flags(task, RETRO_TASK_FLG_FINISHED, true); +} + +static void dbscan_prep_cb(retro_task_t *task, void *task_data, + void *user_data, const char *err) +{ + dbscan_chain_t *chain = (dbscan_chain_t*)user_data; + dbscan_prep_t *prep = (dbscan_prep_t*)task->state; + + if (prep->kind == DBSCAN_PREP_LIST) + { + chain->playlists = prep->list; + chain->all_next = 0; + chain->all_running = !prep->failed; + chain->all_counted = false; + } + else + chain->rdb_list = prep->list; + prep->list = NULL; + if (task_get_flags(task) & RETRO_TASK_FLG_CANCELLED) + { + chain->totals.cancelled = true; + dbscan_chain_finish(chain); + return; + } + if (prep->kind == DBSCAN_PREP_LIST && prep->failed) + { + if (chain->cli_output) + fprintf(stderr, "out of memory\n"); + dbscan_chain_fold(chain, prep->dir, TASK_DBSCAN_FAILED, NULL); + } + dbscan_chain_next(chain); +} + +/* Names ignore case, as the file system may: the name and the path + * come from the folder's listing, the exact name first, since a save + * onto the path as typed can rename the file. Two in other cases are + * ambiguous. False while the listing needs another step. */ +static bool dbscan_load_name_step(dbscan_prep_t *prep) +{ + size_t i; + int r; + unsigned matches = 0; + const char *want = path_basename(prep->config.path); + const char *match = NULL; + + if (task_queue_is_threaded()) + r = dir_list_iter_step(prep->iter, NULL, NULL); + else + { + nbio_budget_t b; + task_nbio_slice_open(&b); + r = dir_list_iter_step(prep->iter, + manual_scan_walk_within_budget, &b); + task_nbio_slice_close(&b); + if (!r) + return false; + } + for (i = 0; r > 0 && i < prep->list->size; i++) + { + const char *path = prep->list->elems[i].data; + + if (string_is_equal(path_basename(path), want)) + { + match = path; + matches = 1; + break; + } + if ( string_is_equal_case_insensitive(path_basename(path), want) + && !matches++) + match = path; + } + if (matches > 1) + { + prep->matches = matches; + prep->load = DBSCAN_LOAD_AMBIGUOUS; + } + else if (match) + { + fill_pathname(prep->name, path_basename(match), "", NAME_MAX_LENGTH); + playlist_config_set_path(&prep->config, match); + } + dir_list_iter_free(prep->iter); + prep->iter = NULL; + return true; +} + +/* @prep loads @path: its name, the path, and, when @list_dir, the + * folder to list for the name's case. Strings only, any thread. */ +static void dbscan_prep_set_playlist(dbscan_prep_t *prep, + const char *path, bool list_dir) +{ + fill_pathname(prep->name, path_basename(path), "", NAME_MAX_LENGTH); + if (list_dir && (prep->dir = (char*)malloc(PATH_MAX_LENGTH))) + { + fill_pathname_basedir(prep->dir, path, PATH_MAX_LENGTH); +#ifdef _WIN32 + pathname_conform_slashes_to_os(prep->dir); +#endif + } + playlist_config_set_path(&prep->config, path); +} + +/* The playlist a --scan-refresh's @prep->arg names in @prep->pl_dir: + * false, with the status set, when it names none there. On the + * load's worker, as the realpaths ask the file system. */ +static bool dbscan_prep_refresh_resolve(dbscan_prep_t *prep) +{ + const char *arg = prep->arg; + size_t _len = strlen(arg); + char *path = NULL; + char *dir = NULL; + char *want = NULL; + bool ok = false; + + /* A playlist name cannot hold a slash; a file name is one too. */ + if (!find_last_slash(arg)) + { + if ((path = (char*)malloc(PATH_MAX_LENGTH))) + { + fill_pathname_join_special(path, prep->pl_dir, arg, + PATH_MAX_LENGTH); + if (!( _len >= STRLEN_CONST(".lpl") + && string_is_equal_case_insensitive( + arg + _len - STRLEN_CONST(".lpl"), ".lpl"))) + strlcat(path, ".lpl", PATH_MAX_LENGTH); + dbscan_prep_set_playlist(prep, path, true); + ok = true; + } + else + prep->load = DBSCAN_LOAD_NO_MEMORY; + free(path); + return ok; + } + path = dbscan_chain_expand(arg); + dir = dbscan_chain_expand(prep->pl_dir); + want = (char*)malloc(PATH_MAX_LENGTH); + if (path && dir && want) + { + /* "//", "/./" and ".." would fail the comparison; @dir stays + * as configured for the line. */ + path_resolve_realpath(path, PATH_MAX_LENGTH, false); + /* The scan writes into the playlist folder, whatever the path. */ + fill_pathname_join_special(want, dir, path_basename(path), + PATH_MAX_LENGTH); + path_resolve_realpath(want, PATH_MAX_LENGTH, false); +#ifdef _WIN32 + pathname_conform_slashes_to_os(path); + pathname_conform_slashes_to_os(want); + if (string_is_equal_case_insensitive(path, want)) +#else + if (string_is_equal(path, want)) +#endif + { + dbscan_prep_set_playlist(prep, path, true); + ok = true; + } + else + { + prep->load = DBSCAN_LOAD_OUTSIDE; + prep->detail = dir; + dir = NULL; + } + } + else + prep->load = DBSCAN_LOAD_NO_MEMORY; + free(path); + free(dir); + free(want); + return ok; +} + +/* Loads one playlist and tells whether it can be refreshed: to the end + * on a threaded queue's worker, else under the shared window. */ +static void dbscan_load_handler(retro_task_t *task) +{ + dbscan_prep_t *prep = (dbscan_prep_t*)task->state; + + if (task_get_flags(task) & RETRO_TASK_FLG_CANCELLED) + { + task_set_flags(task, RETRO_TASK_FLG_FINISHED, true); + return; + } + if (!prep->begun) + { + prep->begun = true; + /* A --scan-refresh: its path first. */ + if (prep->arg && !dbscan_prep_refresh_resolve(prep)) + { + task_set_flags(task, RETRO_TASK_FLG_FINISHED, true); + return; + } + /* The parse waits for the listed path. */ + if (prep->dir && (prep->list = string_list_new())) + prep->iter = dir_list_iter_new(prep->dir, "lpl", false, true, + false, false, prep->list); + if (prep->iter) + return; + if (!path_is_valid(prep->config.path)) + prep->load = DBSCAN_LOAD_NO_PLAYLIST; + else + /* The header alone: the saved settings, never the entries, + * and nothing written. */ + prep->parse = playlist_parse_begin_header(&prep->config); + } + /* One window per call on the main thread. */ + if (prep->iter) + { + if (!dbscan_load_name_step(prep)) + return; + if (prep->load == DBSCAN_LOAD_OK) + { + if (!path_is_valid(prep->config.path)) + prep->load = DBSCAN_LOAD_NO_PLAYLIST; + else + /* The header alone: the saved settings, never the entries, + * and nothing written. */ + prep->parse = playlist_parse_begin_header(&prep->config); + } + if (!task_queue_is_threaded()) + return; + } + if (prep->parse) + { + int r; + + if (task_queue_is_threaded()) + r = playlist_parse_step(prep->parse, NULL, NULL); + else + { + nbio_budget_t b; + task_nbio_slice_open(&b); + r = playlist_parse_step(prep->parse, + manual_scan_walk_within_budget, &b); + task_nbio_slice_close(&b); + if (!r) + return; + } + if (r < 0) + playlist_parse_abort(prep->parse); + else + prep->playlist = playlist_parse_end(prep->parse); + prep->parse = NULL; + } + if (prep->load == DBSCAN_LOAD_OK) + { + /* A playlist that does not parse has no settings to read. */ + if (!playlist_scan_refresh_enabled(prep->playlist)) + prep->load = DBSCAN_LOAD_NO_SETTINGS; + /* Here, so that the refresh asks the file system nothing on the + * main thread: the folder, the saved DAT, the overrides'. */ + else if (!(prep->fs = manual_content_scan_fs_read( + playlist_get_scan_content_dir(prep->playlist), + playlist_get_scan_dat_file_path(prep->playlist), + prep->ov_core, prep->ov_dat))) + prep->load = DBSCAN_LOAD_NO_MEMORY; + /* A single file's scan saves the file. */ + else if (!(prep->fs->content_stat & RETRO_VFS_STAT_IS_VALID)) + prep->load = DBSCAN_LOAD_FOLDER_MISSING; + } + task_set_flags(task, RETRO_TASK_FLG_FINISHED, true); +} + +static void dbscan_load_cb(retro_task_t *task, void *task_data, + void *user_data, const char *err) +{ + dbscan_chain_t *chain = (dbscan_chain_t*)user_data; + dbscan_prep_t *prep = (dbscan_prep_t*)task->state; + const task_dbscan_item_t *item = &chain->items[chain->next - 1]; + /* A --scan-refresh's own failures name it as typed. */ + const char *arg = item->kind == TASK_DBSCAN_ITEM_REFRESH + ? item->arg : prep->name; + + if (task_get_flags(task) & RETRO_TASK_FLG_CANCELLED) + { + chain->totals.cancelled = true; + dbscan_chain_finish(chain); + return; + } + if (prep->load != DBSCAN_LOAD_NO_SETTINGS) + chain->all_counted = true; + switch (prep->load) + { + case DBSCAN_LOAD_NO_PLAYLIST: + dbscan_chain_refresh_error(chain, arg, "no such playlist", NULL); + dbscan_chain_fold(chain, arg, TASK_DBSCAN_FAILED, NULL); + break; + case DBSCAN_LOAD_AMBIGUOUS: + { + char reason[80]; + snprintf(reason, sizeof(reason), + "%u playlists with this name in other cases, " + "give the exact name", + prep->matches); + dbscan_chain_refresh_error(chain, arg, reason, NULL); + dbscan_chain_fold(chain, arg, TASK_DBSCAN_FAILED, NULL); + } + break; + case DBSCAN_LOAD_NO_SETTINGS: + /* A refresh-all neither lists nor counts it. */ + if (item->kind == TASK_DBSCAN_ITEM_REFRESH_ALL) + break; + dbscan_chain_refresh_error(chain, arg, "no saved scan settings", + NULL); + dbscan_chain_fold(chain, arg, TASK_DBSCAN_FAILED, NULL); + break; + case DBSCAN_LOAD_OUTSIDE: + dbscan_chain_refresh_error(chain, arg, + "not in the playlist folder", prep->detail); + dbscan_chain_fold(chain, arg, TASK_DBSCAN_FAILED, NULL); + break; + case DBSCAN_LOAD_NO_MEMORY: + if (chain->cli_output) + fprintf(stderr, "out of memory\n"); + dbscan_chain_fold(chain, arg, TASK_DBSCAN_FAILED, NULL); + break; + case DBSCAN_LOAD_FOLDER_MISSING: + dbscan_chain_refresh_error(chain, prep->name, + "skipped: content folder not found", + playlist_get_scan_content_dir(prep->playlist)); + dbscan_chain_fold(chain, prep->name, TASK_DBSCAN_SKIPPED, NULL); + break; + case DBSCAN_LOAD_OK: + if (dbscan_chain_push_refreshed(chain, item, prep)) + return; + break; + } + dbscan_chain_next(chain); +} + +static void dbscan_prep_state_free(dbscan_prep_t *prep) +{ + if (!prep) + return; + dir_list_iter_free(prep->iter); + if (prep->list) + string_list_free(prep->list); + playlist_parse_abort(prep->parse); + playlist_free(prep->playlist); + manual_content_scan_fs_free(prep->fs); + free(prep->name); + free(prep->dir); + free(prep->arg); + free(prep->pl_dir); + free(prep->ov_core); + free(prep->ov_dat); + free(prep->path); + free(prep->detail); + free(prep); +} + +static void dbscan_prep_free(retro_task_t *task) +{ + dbscan_prep_state_free((dbscan_prep_t*)task->state); +} + +/* Copies @item's core and DAT overrides into @prep, for its worker; + * false out of memory. */ +static bool dbscan_prep_take_overrides(dbscan_prep_t *prep, + const task_dbscan_item_t *item) +{ + const manual_content_scan_overrides_t *ov = item->overrides; + + if (!ov) + return true; + if (ov->core && !(prep->ov_core = strdup(ov->core))) + return false; + if (ov->dat && !(prep->ov_dat = strdup(ov->dat))) + return false; + return true; +} + +/* Pushes a prep task for @prep; false, @prep freed, if it cannot. */ +static bool dbscan_chain_push_prep(dbscan_chain_t *chain, + dbscan_prep_t *prep, retro_task_callback_t cb) +{ + retro_task_t *task = prep ? task_init() : NULL; + + if (!task) + { + dbscan_prep_state_free(prep); + return false; + } + /* Silent: no title, no notifications. Each handler by name, so the + * thread-read audit takes them all as worker entries. */ + switch (prep->kind) + { + case DBSCAN_PREP_LOAD: + task->handler = dbscan_load_handler; + break; + case DBSCAN_PREP_PATH: + task->handler = dbscan_path_handler; + break; + default: + task->handler = dbscan_prep_handler; + break; + } + task->state = prep; + task->user_data = chain; + task->callback = cb; + task->cleanup = dbscan_prep_free; + task_queue_push(task); + return true; +} + +/* A load prep for the current item, its playlist not yet set: NULL + * out of memory. */ +static dbscan_prep_t *dbscan_prep_load_new(dbscan_chain_t *chain) +{ + settings_t *settings = config_get_ptr(); + dbscan_prep_t *prep = (dbscan_prep_t*)calloc(1, sizeof(*prep)); + + if (!prep) + return NULL; + prep->kind = DBSCAN_PREP_LOAD; + if ( !(prep->name = (char*)malloc(NAME_MAX_LENGTH)) + || !dbscan_prep_take_overrides(prep, + &chain->items[chain->next - 1])) + { + dbscan_prep_state_free(prep); + return NULL; + } + *prep->name = '\0'; + { + /* As Manage Playlists opens it. */ + prep->config.capacity = COLLECTION_SIZE; + prep->config.old_format = settings->bools.playlist_use_old_format; + prep->config.compress = settings->bools.playlist_compression; + prep->config.fuzzy_archive_match = settings->bools.playlist_fuzzy_archive_match; + playlist_config_set_base_content_directory(&prep->config, + settings->bools.playlist_portable_paths + ? settings->paths.directory_menu_content : NULL); + } + return prep; +} + +/* Loads a refresh-all's playlist at @path off the main thread; + * otherwise folds it as failed. Its names come from its listing + * already. */ +static bool dbscan_chain_push_load(dbscan_chain_t *chain, + const char *path) +{ + char name[NAME_MAX_LENGTH]; + dbscan_prep_t *prep = dbscan_prep_load_new(chain); + + if (prep) + dbscan_prep_set_playlist(prep, path, false); + if (dbscan_chain_push_prep(chain, prep, dbscan_load_cb)) + return true; + fill_pathname(name, path_basename(path), "", sizeof(name)); + dbscan_chain_refresh_error(chain, name, "the scan could not start", NULL); + dbscan_chain_fold(chain, name, TASK_DBSCAN_FAILED, NULL); + return false; +} + +/* True once @item's playlist is loading; otherwise it is folded as + * failed. */ +static bool dbscan_chain_push_refresh(dbscan_chain_t *chain, + const task_dbscan_item_t *item) +{ + dbscan_prep_t *prep = dbscan_prep_load_new(chain); + + /* The worker finds the playlist: the realpaths ask the file + * system. */ + if ( prep + && ( !(prep->arg = strdup(item->arg)) + || !(prep->pl_dir = strdup( + config_get_ptr()->paths.directory_playlist)))) + { + dbscan_prep_state_free(prep); + prep = NULL; + } + if (dbscan_chain_push_prep(chain, prep, dbscan_load_cb)) + return true; + dbscan_chain_refresh_error(chain, item->arg, "the scan could not start", + NULL); + dbscan_chain_fold(chain, item->arg, TASK_DBSCAN_FAILED, NULL); + return false; +} + +static void dbscan_path_cb(retro_task_t *task, void *task_data, + void *user_data, const char *err) +{ + dbscan_chain_t *chain = (dbscan_chain_t*)user_data; + dbscan_prep_t *prep = (dbscan_prep_t*)task->state; + + if (task_get_flags(task) & RETRO_TASK_FLG_CANCELLED) + { + chain->totals.cancelled = true; + dbscan_chain_finish(chain); + return; + } + if (!dbscan_chain_push_scan(chain, &chain->items[chain->next - 1], prep)) + dbscan_chain_next(chain); +} + +/* True once @item's path is being read; otherwise it is folded as + * failed. */ +static bool dbscan_chain_push_path(dbscan_chain_t *chain, + const task_dbscan_item_t *item) +{ + dbscan_prep_t *prep = (dbscan_prep_t*)calloc(1, sizeof(*prep)); + + if (prep) + { + prep->kind = DBSCAN_PREP_PATH; + if ( !(prep->arg = strdup(item->arg)) + || !dbscan_prep_take_overrides(prep, item)) + { + dbscan_prep_state_free(prep); + prep = NULL; + } + } + if (dbscan_chain_push_prep(chain, prep, dbscan_path_cb)) + return true; + fprintf(stderr, "out of memory\n"); + dbscan_chain_fold(chain, item->arg, TASK_DBSCAN_FAILED, NULL); + return false; +} + +/* Lists the playlist folder off the main thread when a refresh-all is + * reached, so it takes what earlier items wrote. */ +static bool dbscan_chain_push_list(dbscan_chain_t *chain) +{ + settings_t *settings = config_get_ptr(); + dbscan_prep_t *prep = (dbscan_prep_t*)calloc(1, sizeof(*prep)); + + if (prep) + { + prep->kind = DBSCAN_PREP_LIST; + prep->include_hidden = settings->bools.show_hidden_files; + if ( *settings->paths.directory_playlist + && !(prep->dir = strdup(settings->paths.directory_playlist))) + { + free(prep); + prep = NULL; + } + } + return dbscan_chain_push_prep(chain, prep, dbscan_prep_cb); +} + +/* Runs items, and a refresh-all's playlists, until one pushes a task; + * after the last, finishes. */ +static void dbscan_chain_next(dbscan_chain_t *chain) +{ + for (;;) + { + const task_dbscan_item_t *item; + + if (chain->all_running) + { + if ( chain->playlists + && chain->all_next < chain->playlists->size) + { + if (dbscan_chain_push_load(chain, + chain->playlists->elems[chain->all_next++].data)) + return; + continue; + } + if (!chain->all_counted && chain->cli_output) + fprintf(stderr, "no playlists with saved scan settings\n"); + if (chain->playlists) + string_list_free(chain->playlists); + chain->playlists = NULL; + chain->all_running = false; + } + if (chain->next >= chain->count) + break; + item = &chain->items[chain->next++]; + switch (item->kind) + { + case TASK_DBSCAN_ITEM_PATH: + if (dbscan_chain_push_path(chain, item)) + return; + break; + case TASK_DBSCAN_ITEM_REFRESH: + if (dbscan_chain_push_refresh(chain, item)) + return; + break; + case TASK_DBSCAN_ITEM_REFRESH_ALL: + if (dbscan_chain_push_list(chain)) + return; + if (chain->cli_output) + fprintf(stderr, "out of memory\n"); + dbscan_chain_fold(chain, + config_get_ptr()->paths.directory_playlist, + TASK_DBSCAN_FAILED, NULL); + break; + } + } + dbscan_chain_finish(chain); +} + +/* Any unfinished scan, whatever its playlist. */ +static bool dbscan_chain_scan_finder(retro_task_t *task, void *user_data) +{ + return task->handler == task_manual_content_scan_handler + && !(task_get_flags(task) & RETRO_TASK_FLG_FINISHED); +} + +static void dbscan_chain_refusal(void) +{ + const char *_msg = msg_hash_to_str(MSG_MANUAL_CONTENT_SCAN_ALREADY_RUNNING); + runloop_msg_queue_push(_msg, strlen(_msg), 1, 150, true, NULL, + MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_ERROR); +} + +bool task_dbscan_chain_refuse(void) +{ + if (!dbscan_chain_running) + return false; + dbscan_chain_refusal(); + return true; +} + +bool task_dbscan_chain_refuse_start(void) +{ + if (task_dbscan_chain_can_start()) + return false; + dbscan_chain_refusal(); + return true; +} + +bool task_dbscan_chain_can_start(void) +{ + task_finder_data_t find_data; + + find_data.func = dbscan_chain_scan_finder; + find_data.userdata = NULL; + return !dbscan_chain_running && !task_queue_find(&find_data); +} + +bool task_push_dbscan_chain(task_dbscan_item_t *items, size_t count, + task_dbscan_item_cb_t item_cb, task_dbscan_chain_cb_t done_cb, + void *user_data) +{ + dbscan_chain_t *chain = NULL; + dbscan_prep_t *prep = NULL; + settings_t *settings = config_get_ptr(); + + if (!task_dbscan_chain_can_start()) + goto error; + if ( !(chain = (dbscan_chain_t*)calloc(1, sizeof(*chain))) + || !(prep = (dbscan_prep_t*)calloc(1, sizeof(*prep)))) + goto error; +#ifdef HAVE_LIBRETRODB + if ( *settings->paths.path_content_database + && !(prep->dir = strdup(settings->paths.path_content_database))) + goto error; +#endif + prep->kind = DBSCAN_PREP_DBS; + prep->include_hidden = settings->bools.show_hidden_files; + + chain->items = items; + chain->count = count; + chain->item_cb = item_cb; + chain->done_cb = done_cb; + chain->user_data = user_data; + chain->cli_output = retroarch_override_setting_is_set( + RARCH_OVERRIDE_SETTING_DATABASE_SCAN, NULL); + if (!(chain->saved = manual_content_scan_save_settings())) + goto error; + + dbscan_chain_running = chain; + if (dbscan_chain_push_prep(chain, prep, dbscan_prep_cb)) + return true; + dbscan_chain_running = NULL; + prep = NULL; + +error: + dbscan_prep_state_free(prep); + if (chain) + free(chain->saved); + free(chain); + dbscan_chain_free_items(items, count); + return false; +} + +bool task_dbscan_chain_is_running(void) +{ + return dbscan_chain_running != NULL; +} diff --git a/tasks/tasks_internal.h b/tasks/tasks_internal.h index 4d116cdf394c..805da72e559a 100644 --- a/tasks/tasks_internal.h +++ b/tasks/tasks_internal.h @@ -336,16 +336,90 @@ bool task_push_image_load_set(const char *const *paths, * task. Safe from a task_queue_find predicate. */ bool task_image_set_tag(retro_task_t *task, uint64_t *tag); +/* A content scan's outcome, handed to its callback as task_data and + * valid only during the call. Counts cover every playlist the scan + * writes; errors are files it could not read. */ +typedef struct +{ + unsigned added; + unsigned duplicates; + unsigned unmatched; + unsigned errors; + bool failed; +} task_dbscan_result_t; + #ifdef HAVE_LIBRETRODB /* Scans @fullpath, a directory or a single file, against the content * databases; the database and playlist directories come from the - * settings. */ -bool task_push_dbscan(const char *fullpath, retro_task_callback_t cb); + * settings. @cb gets @user_data. False while a chain runs. */ +bool task_push_dbscan(const char *fullpath, retro_task_callback_t cb, + void *user_data); #endif +struct manual_content_scan_overrides; + +enum task_dbscan_item_kind +{ + TASK_DBSCAN_ITEM_PATH = 0, /* --scan */ + TASK_DBSCAN_ITEM_REFRESH, /* --scan-refresh */ + TASK_DBSCAN_ITEM_REFRESH_ALL /* --scan-refresh-all, Refresh All */ +}; + +typedef struct +{ + char *arg; /* as typed; NULL for REFRESH_ALL */ + struct manual_content_scan_overrides *overrides; /* NULL: none */ + enum task_dbscan_item_kind kind; +} task_dbscan_item_t; + +enum task_dbscan_outcome +{ + TASK_DBSCAN_RAN = 0, /* res holds the counts */ + TASK_DBSCAN_FAILED, + TASK_DBSCAN_SKIPPED +}; + +typedef struct +{ + unsigned ran; /* scans that ran and did not fail */ + unsigned added; /* of those, the ones that added an entry */ + unsigned failed; + unsigned skipped; + bool cancelled; +} task_dbscan_chain_result_t; + +typedef void (*task_dbscan_item_cb_t)(const char *name, + enum task_dbscan_outcome outcome, + const task_dbscan_result_t *res, void *user_data); +typedef void (*task_dbscan_chain_cb_t)( + const task_dbscan_chain_result_t *res, void *user_data); + +/* Runs @items in order, one task at a time: each step is pushed from + * the last one's callback, and a cancelled task ends the run. The + * chain owns @items from the call, whatever it returns. @item_cb: once + * per scan, failed item or skipped playlist; @done_cb: once at the end, + * from a task callback. @res is valid during the call only. False, + * calling neither, if out of memory or while a chain or a manual scan + * runs. Main thread only. */ +bool task_push_dbscan_chain(task_dbscan_item_t *items, size_t count, + task_dbscan_item_cb_t item_cb, task_dbscan_chain_cb_t done_cb, + void *user_data); + +/* Main thread only. */ +bool task_dbscan_chain_is_running(void); + +/* False while task_push_dbscan_chain() would refuse: a chain or a + * manual scan runs. Main thread only. */ +bool task_dbscan_chain_can_start(void); +/* A scan asked for while a chain runs would use the settings the chain + * last loaded: true, with the message shown, when it must be refused. */ +bool task_dbscan_chain_refuse(void); +/* The same for a new chain, which also waits for a running scan. */ +bool task_dbscan_chain_refuse_start(void); + bool task_push_manual_content_scan( bool do_menu_refresh, - retro_task_callback_t user_cb); + retro_task_callback_t user_cb, void *user_data); #ifdef HAVE_OVERLAY bool task_push_overlay_load_default( diff --git a/tools/thread_read_audit.py b/tools/thread_read_audit.py index 6b6ba2a179b5..92c289e0dfe7 100755 --- a/tools/thread_read_audit.py +++ b/tools/thread_read_audit.py @@ -292,7 +292,7 @@ def enclosing_function(text, pos): return text[start if start >= 0 else 0:end if end >= 0 else len(text)] -def source_entries(root, main_thread=None): +def source_entries(root, main_thread=None, forwarded=None): entries = {} # handler name -> [path, every assignment flagged main-thread] handlers = {} @@ -320,6 +320,14 @@ def source_entries(root, main_thread=None): entries.setdefault(m.group(1), rel) clears = MAIN_CLEAR_RE.search(text) is not None for m in HANDLER_RE.finditer(text): + # A handler set from a retro_task_handler_t parameter names + # no function, and the handlers its callers pass are never + # entries: the audit cannot see them, so it says so. + if re.search(r"\bretro_task_handler_t\s+%s\b" + % re.escape(m.group(1)), text): + if forwarded is not None: + forwarded.append((rel, m.group(1))) + continue main = (not clears and MAIN_SET_RE.search( enclosing_function(text, m.start())) is not None) site = handlers.setdefault(m.group(1), [rel, True]) @@ -417,7 +425,8 @@ def load_allow(path): def run(binary, root, allow_path, list_unaudited=False, objdump="objdump"): main_thread = {} - entries = source_entries(root, main_thread) + forwarded = [] + entries = source_entries(root, main_thread, forwarded) try: out = subprocess.run([objdump, "-d", binary], capture_output=True, text=True, check=True) @@ -436,6 +445,12 @@ def run(binary, root, allow_path, list_unaudited=False, print("tools/thread_read_allow.list.") for entry, fn, reader in findings: print("%s -> %s calls %s" % (entry, fn, reader)) + if forwarded: + print("%d task handler(s) set from a parameter, so the handlers" + % len(forwarded)) + print("passed in are not audited. Assign each by name:") + for rel, name in forwarded: + print(" %s: task->handler = %s" % (rel, name)) if list_unaudited: absent = sorted(e for e in entries if e not in defined) print("%d source entr%s absent from this binary - compiled out" @@ -452,7 +467,7 @@ def run(binary, root, allow_path, list_unaudited=False, len(boundaries), "y" if len(boundaries) == 1 else "ies", len(main_thread)), file=sys.stderr) - return 1 if findings else 0 + return 1 if findings or forwarded else 0 FIXTURE = """ @@ -520,11 +535,22 @@ def selftest(): " task_set_flags(task, RETRO_TASK_FLG_MAIN_THREAD," " false);\n" "}\n") + # A helper that sets whatever handler it is given hides it. + with open(os.path.join(td, "fwd.c"), "w") as f: + f.write("static void push_any(retro_task_t *task,\n" + " retro_task_handler_t handler)\n" + "{\n" + " task->handler = handler;\n" + "}\n") if subprocess.run(["gcc", "-O0", src, "-o", binp, "-lpthread"]).returncode: print("selftest: fixture build failed") return 1 - entries = source_entries(td) + forwarded = [] + entries = source_entries(td, None, forwarded) + if forwarded != [("fwd.c", "handler")] or "handler" in entries: + print("selftest: FAIL forwarded handler %r" % forwarded) + return 1 out = subprocess.run(["objdump", "-d", binp], capture_output=True, text=True, check=True) calls, defined = call_graph(out.stdout.splitlines()) diff --git a/ui/companion/companion_core.c b/ui/companion/companion_core.c index 3a52e3868f6a..e29fb41fd80f 100644 --- a/ui/companion/companion_core.c +++ b/ui/companion/companion_core.c @@ -81,8 +81,9 @@ static void companion_core_download_orphan(companion_core_t *core); #endif #ifdef HAVE_LIBRETRODB -/* task_push_dbscan() carries no user data, so the requesting core is - * kept here. Only one desktop companion is active at a time. */ +/* The requesting core, kept here rather than passed as the scan's + * user data so that freeing the core clears it. Only one desktop + * companion is active at a time. */ static companion_core_t *companion_core_scan_owner = NULL; #endif @@ -1848,6 +1849,15 @@ static void companion_core_scan_finished(retro_task_t *task, } #endif +const char *companion_core_scan_refusal(void) +{ +#ifdef HAVE_LIBRETRODB + if (task_dbscan_chain_is_running()) + return msg_hash_to_str(MSG_MANUAL_CONTENT_SCAN_ALREADY_RUNNING); +#endif + return "Scanning is not available in this build."; +} + bool companion_core_request_scan(companion_core_t *core, const char *path) { #ifdef HAVE_LIBRETRODB @@ -1855,7 +1865,7 @@ bool companion_core_request_scan(companion_core_t *core, const char *path) return false; companion_core_scan_owner = core; - return task_push_dbscan(path, companion_core_scan_finished); + return task_push_dbscan(path, companion_core_scan_finished, NULL); #else (void)core; (void)path; diff --git a/ui/companion/companion_core.h b/ui/companion/companion_core.h index 4965a4deb080..e3d3a00e9c23 100644 --- a/ui/companion/companion_core.h +++ b/ui/companion/companion_core.h @@ -272,6 +272,8 @@ void companion_core_event_command(companion_core_t *core, * list has been reset. Returns false when no scan could be started * (no libretrodb in this build, or the task could not be pushed). */ bool companion_core_request_scan(companion_core_t *core, const char *path); +/* Why companion_core_request_scan() refused, for the status line. */ +const char *companion_core_scan_refusal(void); /* --- Companion settings (retroarch.cfg) ------------------------------- */ diff --git a/ui/companion/test/companion_core_stubs.c b/ui/companion/test/companion_core_stubs.c index d4c651c954a4..9a052c797435 100644 --- a/ui/companion/test/companion_core_stubs.c +++ b/ui/companion/test/companion_core_stubs.c @@ -209,14 +209,18 @@ bool task_push_load_content_with_current_core_from_companion_ui( bool task_push_start_current_core(content_ctx_info_t *ci) { (void)ci; stub_calls_start_core++; return true; } -bool task_push_dbscan(const char *path, retro_task_callback_t cb) +bool task_push_dbscan(const char *path, retro_task_callback_t cb, + void *user_data) { (void)cb; + (void)user_data; stub_calls_dbscan++; strlcpy(stub_last_scan_dir, path ? path : "", sizeof(stub_last_scan_dir)); return true; } +bool task_dbscan_chain_is_running(void) { return false; } + /* Entries and macros (input_driver.c): kept by number as their text. An * entry reads when it has a colon, a macro when it has a digit, as a * stand-in for the real parsers; sixteen of each at most. */ diff --git a/ui/drivers/ui_cocoa_companion.m b/ui/drivers/ui_cocoa_companion.m index d30c788eacd3..3b9cb100365e 100644 --- a/ui/drivers/ui_cocoa_companion.m +++ b/ui/drivers/ui_cocoa_companion.m @@ -4227,7 +4227,7 @@ - (void)scanDirectory:(id)sender if (companion_core_request_scan(wimp->core, [path UTF8String])) [self setStatus:"Scanning..."]; else - [self setStatus:"Scanning is not available in this build."]; + [self setStatus:companion_core_scan_refusal()]; } /* The companion's own Load Core window (like Qt's LoadCoreWindow): diff --git a/ui/drivers/ui_cocoatouch.m b/ui/drivers/ui_cocoatouch.m index 203b11560b89..ead1f899e36c 100644 --- a/ui/drivers/ui_cocoatouch.m +++ b/ui/drivers/ui_cocoatouch.m @@ -1507,7 +1507,7 @@ -(BOOL)application:(UIApplication *)app openURL:(NSURL *)url options:(NSDictiona sel_registerName("stopAccessingSecurityScopedResource")); } } - task_push_dbscan(fullpath, NULL); + task_push_dbscan(fullpath, NULL, NULL); return true; } diff --git a/ui/drivers/ui_win32_companion.c b/ui/drivers/ui_win32_companion.c index 2834b8308687..b05b7181d2c6 100644 --- a/ui/drivers/ui_win32_companion.c +++ b/ui/drivers/ui_win32_companion.c @@ -4606,7 +4606,7 @@ static void cw_scan_directory(ui_companion_win32_wimp_t *w) if (companion_core_request_scan(w->core, dir)) cw_status_set(w, "Scanning..."); else - cw_status_set(w, "Scanning is not available in this build."); + cw_status_set(w, companion_core_scan_refusal()); } static void cw_context_menu(ui_companion_win32_wimp_t *w, HWND from,