From bdf1906060628389030467da8019ecdf2db162c4 Mon Sep 17 00:00:00 2001 From: davey Date: Fri, 30 Oct 2026 15:35:09 -0700 Subject: [PATCH 01/21] file_path: stop at the end of the path after '~' fill_pathname_expand_special() skipped two characters after a leading '~' or ':', so a bare "~" read past its NUL and "~x" lost the x. It now skips the prefix and a slash after it, if there is one. --- libretro-common/file/file_path.c | 4 +- .../file/file_path/fill_pathname_test.c | 58 ++++++++++++++++++- 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/libretro-common/file/file_path.c b/libretro-common/file/file_path.c index 429baf04962b..88734e0d8e26 100644 --- a/libretro-common/file/file_path.c +++ b/libretro-common/file/file_path.c @@ -1147,7 +1147,9 @@ size_t fill_pathname_expand_special(char *s, const char *in_path, size_t len) len -= _len; } - in_path += 2; + in_path++; + if (PATH_CHAR_IS_SLASH(*in_path)) + in_path++; } } #endif diff --git a/libretro-common/samples/file/file_path/fill_pathname_test.c b/libretro-common/samples/file/file_path/fill_pathname_test.c index 4757476c8a37..b481f76b0df7 100644 --- a/libretro-common/samples/file/file_path/fill_pathname_test.c +++ b/libretro-common/samples/file/file_path/fill_pathname_test.c @@ -259,6 +259,16 @@ static void test_join_truncation(void) free(region); } +/* mingw-w64 has no setenv(). */ +static void set_home(const char *home) +{ +#ifdef _WIN32 + _putenv_s("HOME", home); +#else + setenv("HOME", home, 1); +#endif +} + /* fill_pathname_abbreviated_or_relative() resolves a relative @in_path * against @in_refpath's directory before choosing between the * relative and abbreviated forms. That join still carries its '..' @@ -273,7 +283,7 @@ static void test_abbreviated_or_relative_long_join(void) size_t _len = 0; size_t i; - setenv("HOME", "/nonexistent-home", 1); + set_home("/nonexistent-home"); _len = strlcpy(refpath, "/r/s/t/u/v/w/", sizeof(refpath)); for (i = 0; i < 230; i++) @@ -315,6 +325,51 @@ static void test_abbreviated_or_relative_long_join(void) printf("[SUCCESS] absolute reference preferred over deeper relative\n"); } +/* Equal but for which slash separates. */ +static int path_same(const char *a, const char *b) +{ + for (; *a && *b; a++, b++) + if (*a != *b && !(PATH_CHAR_IS_SLASH(*a) && PATH_CHAR_IS_SLASH(*b))) + return 0; + return *a == *b; +} + +/* fill_pathname_expand_special() replaces '~' and the slash after it. + * The input is a heap copy, so ASan sees a read past its NUL. */ +static void test_expand_special_home(void) +{ + static const char *const in[] = + { "~", "~/", "~/x", "~x", "", "a/~" }; + static const char *const want[] = + { "/h/", "/h/", "/h/x", "/h/x", "", "a/~" }; + char out[64]; + size_t i; + + set_home("/h"); + for (i = 0; i < sizeof(in) / sizeof(in[0]); i++) + { + size_t n = strlen(in[i]) + 1; + char *path = (char*)malloc(n); + if (!path) + { + printf("[FAILED] expand \"%s\": out of memory\n", in[i]); + failures++; + continue; + } + memcpy(path, in[i], n); + fill_pathname_expand_special(out, path, sizeof(out)); + free(path); + if (!path_same(out, want[i])) + { + printf("[FAILED] expand \"%s\" gave \"%s\" (expected \"%s\")\n", + in[i], out, want[i]); + failures++; + } + else + printf("[SUCCESS] expand \"%s\"\n", in[i]); + } +} + int main(void) { /* Documented semantics. */ @@ -335,6 +390,7 @@ int main(void) test_overlong_input_no_dot(); test_join_truncation(); test_abbreviated_or_relative_long_join(); + test_expand_special_home(); if (failures) { From 0774150f091056eae28bafeef167e0400284e414 Mon Sep 17 00:00:00 2001 From: davey Date: Fri, 30 Oct 2026 15:43:24 -0700 Subject: [PATCH 02/21] scan: hand the callback user data and what the scan did task_push_dbscan() and task_push_manual_content_scan() take user data for their callback, and the callback's task_data is now a task_dbscan_result_t: how many entries the scan added, how many its playlists already held, and how many files matched nothing. A file is counted by what became of it, so an archive that is expanded is not counted as unmatched, and a loose scan's unmatched files count as added. Callers pass NULL for now. --- menu/cbs/menu_cbs_ok.c | 4 +- menu/cbs/menu_cbs_scan.c | 4 +- retroarch.c | 2 +- samples/tasks/database/database_scan_test.c | 214 +++++++++++++++++- samples/tasks/database/main.c | 2 +- .../tasks/database/scan_begin_budget_test.c | 8 +- tasks/task_database.c | 59 ++++- tasks/tasks_internal.h | 18 +- ui/companion/companion_core.c | 7 +- ui/companion/test/companion_core_stubs.c | 4 +- ui/drivers/ui_cocoatouch.m | 2 +- 11 files changed, 290 insertions(+), 34 deletions(-) diff --git a/menu/cbs/menu_cbs_ok.c b/menu/cbs/menu_cbs_ok.c index 53a916a5b9e4..c391cbbf49bf 100644 --- a/menu/cbs/menu_cbs_ok.c +++ b/menu/cbs/menu_cbs_ok.c @@ -9133,7 +9133,7 @@ static int action_ok_manual_content_scan_start(const char *path, settings->bools.playlist_portable_paths ? settings->paths.directory_menu_content : NULL); - task_push_manual_content_scan(true, NULL); + task_push_manual_content_scan(true, NULL, NULL); return 0; } @@ -9683,7 +9683,7 @@ static int action_ok_playlist_refresh(const char *path, settings->bools.playlist_portable_paths ? settings->paths.directory_menu_content : NULL); - task_push_manual_content_scan(false, NULL); + task_push_manual_content_scan(false, NULL, NULL); } return 0; } diff --git a/menu/cbs/menu_cbs_scan.c b/menu/cbs/menu_cbs_scan.c index 565346fd2463..517d1a60b0c9 100644 --- a/menu/cbs/menu_cbs_scan.c +++ b/menu/cbs/menu_cbs_scan.c @@ -69,7 +69,7 @@ int action_scan_file(const char *path, fill_pathname_join_special(fullpath, menu_path, path, sizeof(fullpath)); - task_push_dbscan(fullpath, handle_dbscan_finished); + task_push_dbscan(fullpath, handle_dbscan_finished, NULL); return 0; } @@ -96,7 +96,7 @@ int action_scan_directory(const char *path, else strlcpy(fullpath, menu_path, sizeof(fullpath)); - task_push_dbscan(fullpath, handle_dbscan_finished); + task_push_dbscan(fullpath, handle_dbscan_finished, NULL); return 0; } diff --git a/retroarch.c b/retroarch.c index d0606c069d4d..ae7abe4af6bb 100644 --- a/retroarch.c +++ b/retroarch.c @@ -9048,7 +9048,7 @@ static bool retroarch_parse_input_and_config( cb_task_dbscan = handle_dbscan_finished; #endif - task_push_dbscan(optarg, cb_task_dbscan); + task_push_dbscan(optarg, cb_task_dbscan, NULL); if (!explicit_menu) { diff --git a/samples/tasks/database/database_scan_test.c b/samples/tasks/database/database_scan_test.c index 85a3d864c219..77e5187434cb 100644 --- a/samples/tasks/database/database_scan_test.c +++ b/samples/tasks/database/database_scan_test.c @@ -1889,13 +1889,18 @@ static int file_contains(const char *path, const char *needle) static bool loop_active = true; static bool scan_completed = false; static retro_time_t cb_usec; +/* What the last run_scan() was handed: its user data is this. */ +static task_dbscan_result_t scan_result; static void scan_cb(retro_task_t *task, void *task_data, void *user_data, const char *err) { - (void)task; (void)task_data; (void)user_data; + (void)task; if (err && *err) fprintf(stderr, " scan reported an error: %s\n", err); + if (user_data && task_data) + *(task_dbscan_result_t*)user_data = + *(const task_dbscan_result_t*)task_data; retro_atomic_store_release_int(&cb_fired, 1); cb_usec = cpu_features_get_time_usec(); scan_completed = true; @@ -1927,7 +1932,12 @@ static bool run_scan(const char *dir) retro_atomic_store_release_int(&rd_watch_wholes, 0); retro_atomic_store_release_int(&look_hits, 0); retro_atomic_store_release_int(&look_late, 0); - if (!task_push_dbscan(dir, scan_cb)) + /* Kept unless the callback gets both pointers. */ + scan_result.added = 9999; + scan_result.duplicates = 9999; + scan_result.unmatched = 9999; + scan_result.failed = true; + if (!task_push_dbscan(dir, scan_cb, &scan_result)) return false; started = time(NULL); @@ -1954,7 +1964,7 @@ static unsigned run_scan_counted(const char *dir) loop_active = true; scan_completed = false; - if (!task_push_dbscan(dir, scan_cb)) + if (!task_push_dbscan(dir, scan_cb, NULL)) return 0; started = time(NULL); @@ -2306,6 +2316,191 @@ static void single_lane(const char *root, const char *pl_dir, free(kept); } +/* Checks the result the last run_scan() was handed. */ +static void check_result(const char *what, unsigned added, + unsigned duplicates, unsigned unmatched, bool failed) +{ + char detail[128]; + + sprintf(detail, "%u added, %u duplicates, %u unmatched%s", + scan_result.added, scan_result.duplicates, scan_result.unmatched, + scan_result.failed ? ", failed" : ""); + check( scan_completed + && scan_result.added == added + && scan_result.duplicates == duplicates + && scan_result.unmatched == unmatched + && scan_result.failed == failed, + what, scan_completed ? detail : "refused or timed out"); +} + +/* Deletes the playlists an earlier run left in @dir. */ +static void delete_playlists(const char *dir) +{ + size_t i; + struct string_list *l = dir_list_new(dir, "lpl", false, false, false, + false); + + if (!l) + return; + for (i = 0; i < l->size; i++) + filestream_delete(l->elems[i].data); + string_list_free(l); +} + +/* A scan counts each file by what became of it, across every playlist + * it writes: an archive is not unmatched for the "??" it prints before + * it is expanded, and a loose scan adds what matches nothing. */ +static void counts_lane(const char *root, const char *pl_dir, + uint32_t crc_a, uint32_t sz_a, uint32_t crc_d, uint32_t sz_d) +{ + char *dir = (char*)malloc(512); + char *dat_dir = (char*)malloc(512); + char *m3u_dir = (char*)malloc(512); + char *res_pl = (char*)malloc(512); + char *p = (char*)malloc(1024); + uint8_t *d = (uint8_t*)malloc(sz_a > sz_d ? sz_a : sz_d); + uint32_t i; + FILE *f; + + if (!dir || !dat_dir || !m3u_dir || !res_pl || !p || !d) + { + check(0, "counts lane", "out of memory"); + goto end; + } + + sprintf(dir, "%s/scan_result", root); + sprintf(dat_dir, "%s/scan_result_dat", root); + sprintf(m3u_dir, "%s/scan_result_m3u", root); + sprintf(res_pl, "%s/scan_result_pl", root); + path_mkdir(dir); + path_mkdir(dat_dir); + path_mkdir(m3u_dir); + path_mkdir(res_pl); + + sprintf(p, "%s/01_alpha.bin", dir); + if (!write_content(p, crc_a, sz_a)) + goto fixture; + sprintf(p, "%s/02_none.bin", dir); + if (!write_content(p, 0x0BADF00Du, 1024)) + goto fixture; + /* Matched by the CRC its member stores. */ + for (i = 0; i < sz_d - 4; i++) + d[i] = (uint8_t)((i * 53 + 7) & 0xFF); + crc_force(d, sz_d - 4, crc_d, d + sz_d - 4); + sprintf(p, "%s/03_bundle.zip", dir); + if (!write_zip(p, "inner.bin", d, sz_d, crc_d)) + goto fixture; + /* Its member stores no CRC: expanded, and the member matches + * nothing. */ + for (i = 0; i < sz_a - 4; i++) + d[i] = (uint8_t)((i * 31 + 5) & 0xFF); + crc_force(d, sz_a - 4, crc_a, d + sz_a - 4); + sprintf(p, "%s/04_expand.zip", dir); + if (!write_zip(p, "inner.bin", d, sz_a, 0)) + goto fixture; + + sprintf(p, "%s/01_alpha.bin", dat_dir); + if (!write_content(p, crc_a, sz_a)) + goto fixture; + sprintf(p, "%s/02_none.bin", dat_dir); + if (!write_content(p, 0x0BADF00Du, 1024)) + goto fixture; + sprintf(p, "%s/Game (Disc 1).bin", m3u_dir); + if (!write_content(p, crc_a, sz_a)) + goto fixture; + sprintf(p, "%s/Game (Disc 2).bin", m3u_dir); + if (!write_content(p, crc_a, sz_a)) + goto fixture; + sprintf(p, "%s/Game.m3u", m3u_dir); + if (!(f = fopen(p, "w"))) + goto fixture; + fputs("Game (Disc 1).bin\nGame (Disc 2).bin\n", f); + fclose(f); + + sprintf(p, "%s/scan_result.dat", root); + if (!(f = fopen(p, "w"))) + goto fixture; + fputs("\n\n" + "
Result
\n" + "Alpha\n" + "
\n", f); + fclose(f); + + delete_playlists(res_pl); + strlcpy(config_get_ptr()->paths.directory_playlist, res_pl, + sizeof(config_get_ptr()->paths.directory_playlist)); + + run_scan(dir); + check_result("a scan counts what it adds and what matches nothing", + 2, 0, 2, false); + run_scan(dir); + check_result("a rescan counts what it already has", 0, 2, 2, false); + + delete_playlists(res_pl); + manual_content_scan_set_menu_scan_method( + MANUAL_CONTENT_SCAN_METHOD_CUSTOM); + manual_content_scan_set_menu_scan_use_db( + MANUAL_CONTENT_SCAN_USE_DB_LOOSE); + run_scan(dir); + check_result("a loose scan adds what matches nothing", 4, 0, 0, false); + + delete_playlists(res_pl); + manual_content_scan_set_menu_scan_use_db( + MANUAL_CONTENT_SCAN_USE_DB_DAT_STRICT); + strlcpy(manual_content_scan_get_dat_file_path_ptr(), p, + manual_content_scan_get_dat_file_path_size()); + run_scan(dat_dir); + check_result("a DAT scan counts what the DAT leaves out", 1, 0, 1, false); + + manual_content_scan_get_dat_file_path_ptr()[0] = '\0'; + manual_content_scan_set_menu_scan_use_db( + MANUAL_CONTENT_SCAN_USE_DB_STRICT); + manual_content_scan_set_menu_scan_method( + MANUAL_CONTENT_SCAN_METHOD_AUTOMATIC); + + /* An m3u counts once, as the one entry its discs fold into. */ + delete_playlists(res_pl); + run_scan(m3u_dir); + check_result("an m3u is added, not also unmatched", 1, 0, 0, false); + run_scan(m3u_dir); + check_result("an m3u rescan is one duplicate", 0, 1, 0, false); + + /* One playlist, so the loose scan's own entry for the m3u and the + * one its discs fold into meet there. */ + delete_playlists(res_pl); + manual_content_scan_set_menu_scan_method( + MANUAL_CONTENT_SCAN_METHOD_CUSTOM); + manual_content_scan_set_menu_scan_use_db( + MANUAL_CONTENT_SCAN_USE_DB_LOOSE); + manual_content_scan_set_menu_system_name( + MANUAL_CONTENT_SCAN_SYSTEM_NAME_CONTENT_DIR, NULL); + run_scan(m3u_dir); + check_result("a loose m3u is added, not also a duplicate", + 1, 0, 0, false); + run_scan(m3u_dir); + check_result("a loose m3u rescan is one duplicate", 0, 1, 0, false); + manual_content_scan_set_menu_system_name( + MANUAL_CONTENT_SCAN_SYSTEM_NAME_AUTO, NULL); + manual_content_scan_set_menu_scan_use_db( + MANUAL_CONTENT_SCAN_USE_DB_STRICT); + manual_content_scan_set_menu_scan_method( + MANUAL_CONTENT_SCAN_METHOD_AUTOMATIC); + + strlcpy(config_get_ptr()->paths.directory_playlist, pl_dir, + sizeof(config_get_ptr()->paths.directory_playlist)); + goto end; + +fixture: + check(0, "fixture", "could not write the counts lane"); +end: + free(dir); + free(dat_dir); + free(m3u_dir); + free(res_pl); + free(p); + free(d); +} + /* The queue the next scans run on: threaded where threads are built, * else the regular one, so every build runs the same checks. */ static void use_queue(bool threaded) @@ -2344,7 +2539,7 @@ static retro_time_t run_scan_cancelled(const char *dir, bool on_stuck) loop_active = true; scan_completed = false; - if (!task_push_dbscan(dir, scan_cb)) + if (!task_push_dbscan(dir, scan_cb, NULL)) return -1; finder.func = find_scan; finder.userdata = NULL; @@ -2653,7 +2848,7 @@ static void reader_quit_lane(const char *root, const char *db_dir) slow_reset(READER_STUCK_SIZE, READER_QUIT_MS); loop_active = true; scan_completed = false; - if (!task_push_dbscan(dir, scan_cb)) + if (!task_push_dbscan(dir, scan_cb, NULL)) { check(0, "quit: scan started", "task_push_dbscan refused"); return; @@ -5123,7 +5318,7 @@ static void run_scan_cancel_at(const char *dir, const step_count_t *c, loop_active = true; scan_completed = false; - if (!task_push_dbscan(dir, scan_cb)) + if (!task_push_dbscan(dir, scan_cb, NULL)) return; finder.func = find_scan; finder.userdata = NULL; @@ -7404,7 +7599,7 @@ static retro_time_t run_scan_switching(const char *dir, bool *started, scan_completed = false; *stalls = 0; *switches = 0; - if (!(*started = task_push_dbscan(dir, scan_cb))) + if (!(*started = task_push_dbscan(dir, scan_cb, NULL))) return 0; #ifdef HAVE_THREADS finder.func = find_scan; @@ -7573,7 +7768,7 @@ static void switch_stuck_lane(const char *root, const char *pl_dir, slow_reset(READER_STUCK_SIZE, READER_STUCK_MS); loop_active = true; scan_completed = false; - if (!task_push_dbscan(dir, scan_cb)) + if (!task_push_dbscan(dir, scan_cb, NULL)) { check(0, "a switch past a stuck read: scan completes", "task_push_dbscan refused"); @@ -7796,7 +7991,7 @@ int main(int argc, char **argv) fprintf(f, "display_name = \"Scan Test\"\n" "corename = \"ScanTest\"\n" - "supported_extensions = \"bin|cue|gdi|zip\"\n" + "supported_extensions = \"bin|cue|gdi|m3u|zip\"\n" "database = \"Test Alpha|Test Beta|Test Disc|Test Zip\"\n"); fclose(f); } @@ -8747,6 +8942,7 @@ int main(int argc, char **argv) walk_lane(root, pl_dir, db_dir); single_lane(root, pl_dir, crc_a, sz_a); + counts_lane(root, pl_dir, crc_a, sz_a, crc_d, sz_d); { char reader_db[512]; diff --git a/samples/tasks/database/main.c b/samples/tasks/database/main.c index 3d6b7b0971bd..e74be51459a7 100644 --- a/samples/tasks/database/main.c +++ b/samples/tasks/database/main.c @@ -239,7 +239,7 @@ int main(int argc, char *argv[]) /* The return value matters: false means no task exists, so nothing * will ever call main_db_cb and the loop below would never end. */ - if (!task_push_dbscan(input_dir, main_db_cb)) + if (!task_push_dbscan(input_dir, main_db_cb, NULL)) { fprintf(stderr, "task_push_dbscan refused to start a scan\n"); goto done; diff --git a/samples/tasks/database/scan_begin_budget_test.c b/samples/tasks/database/scan_begin_budget_test.c index d5408da5d8f4..e348661f919b 100644 --- a/samples/tasks/database/scan_begin_budget_test.c +++ b/samples/tasks/database/scan_begin_budget_test.c @@ -527,7 +527,7 @@ static bool run_scan(const char *content_dir, const char *dat_path, return false; t0 = cpu_features_get_time_usec(); - if (!task_push_manual_content_scan(false, scan_cb)) + if (!task_push_manual_content_scan(false, scan_cb, NULL)) { fprintf(stderr, "task_push_manual_content_scan refused\n"); return false; @@ -628,7 +628,7 @@ static bool run_db_scan(const char *content_dir, const char *db_dir, return false; t0 = cpu_features_get_time_usec(); - if (!task_push_manual_content_scan(false, scan_cb)) + if (!task_push_manual_content_scan(false, scan_cb, NULL)) { fprintf(stderr, "database scan: push refused\n"); return false; @@ -693,7 +693,7 @@ static bool run_scan_threaded(const char *content_dir, const char *dat_path, task_queue_init(true, msgq_push); - if (!task_push_manual_content_scan(false, scan_cb)) + if (!task_push_manual_content_scan(false, scan_cb, NULL)) { fprintf(stderr, "threaded: task_push_manual_content_scan refused\n"); task_queue_deinit(); @@ -742,7 +742,7 @@ static bool run_scan_cancelled(const char *content_dir, const char *dat_path, task_queue_init(false, msgq_push); - if (!task_push_manual_content_scan(false, scan_cb)) + if (!task_push_manual_content_scan(false, scan_cb, NULL)) { fprintf(stderr, "cancel lane: push refused\n"); task_queue_deinit(); diff --git a/tasks/task_database.c b/tasks/task_database.c index d648aa66fb79..40807789973f 100644 --- a/tasks/task_database.c +++ b/tasks/task_database.c @@ -139,6 +139,16 @@ static bool scan_results_add(scan_results_t *sr, return true; } +static scan_result_t *scan_results_find(scan_results_t *sr, + const char *entry_path) +{ + size_t i; + for (i = 0; i < sr->count; i++) + if (string_is_equal(sr->results[i].entry_path, entry_path)) + return &sr->results[i]; + return NULL; +} + static void scan_results_free(scan_results_t *sr) { size_t i; @@ -592,6 +602,9 @@ typedef struct manual_scan_handle * this is how a task_push_dbscan caller learns the scan finished; * see cb_task_manual_content_scan. */ retro_task_callback_t user_cb; + void *user_data; + /* The callback's task_data. */ + task_dbscan_result_t result; } manual_scan_handle_t; enum scan_verdict @@ -916,6 +929,7 @@ static void task_database_iterate_m3u( char first_matched_crc[128]; char collapsed_title[NAME_MAX_LENGTH]; rm3u_t *m3u = NULL; + scan_result_t *own; first_matched_db[0] = '\0'; first_matched_crc[0] = '\0'; @@ -925,6 +939,8 @@ static void task_database_iterate_m3u( if (!(m3u = rm3u_load_filestream(m3u_path))) { RARCH_ERR("[Scanner] Failed to open M3U file: \"%s\".\n", m3u_path); + if (!scan_results_find(&_db->scan_results, m3u_path)) + _db->result.unmatched++; return; } @@ -1013,9 +1029,19 @@ static void task_database_iterate_m3u( rm3u_free(m3u); + own = scan_results_find(&_db->scan_results, m3u_path); + /* If we found at least one match, add M3U entry */ if (found_match) { + /* The content pass's entry for the m3u itself would only be + * replaced by this one in the same playlist. */ + if (own && ( *_db->task_config->playlist_file + || string_is_equal(own->db_name, first_matched_db))) + { + free(own->entry_path); + own->entry_path = NULL; + } if (!scan_results_add(&_db->scan_results, m3u_path, collapsed_title, first_matched_crc, first_matched_db, NULL)) { @@ -1027,6 +1053,8 @@ static void task_database_iterate_m3u( collapsed_title, first_matched_db); } } + else if (!own) + _db->result.unmatched++; /* Compact scan_results to remove NULL entries */ { @@ -4084,6 +4112,11 @@ static bool manual_scan_end_flush_tick( * playlist for the same answer. */ playlist_push_unchecked(manual_scan->flush_playlist, &entry); manual_scan->flush_added++; + /* A present m3u is replaced, not added. */ + if (entry_present) + manual_scan->result.duplicates++; + else + manual_scan->result.added++; RARCH_LOG("[Scanner] Add \"%s / %s\".\n", db_name_noext, result->entry_label); @@ -4092,9 +4125,13 @@ static bool manual_scan_end_flush_tick( db_name_noext, true); } /* Entry already exists - output duplicate indicator for CLI scans */ - else if (manual_scan->flush_playlist && manual_scan->task_config->cli_scan_output) - task_database_scan_console_output(result->entry_label, - db_name_noext, false); + else if (manual_scan->flush_playlist) + { + manual_scan->result.duplicates++; + if (manual_scan->task_config->cli_scan_output) + task_database_scan_console_output(result->entry_label, + db_name_noext, false); + } manual_scan->flush_pos++; } @@ -4128,10 +4165,11 @@ static bool manual_scan_end_flush_tick( } #ifdef HAVE_LIBRETRODB -bool task_push_dbscan(const char *fullpath, retro_task_callback_t cb) +bool task_push_dbscan(const char *fullpath, retro_task_callback_t cb, + void *user_data) { manual_content_scan_set_menu_content_dir(fullpath); - return task_push_manual_content_scan(false, cb); + return task_push_manual_content_scan(false, cb, user_data); } #endif @@ -4393,7 +4431,8 @@ static void cb_task_manual_content_scan( * documented as menu-only and the sample in samples/tasks/database * hangs on exactly this. */ if (manual_scan && manual_scan->user_cb) - manual_scan->user_cb(task, task_data, user_data, err); + manual_scan->user_cb(task, &manual_scan->result, + manual_scan->user_data, err); #if defined(HAVE_MENU) /* When creating playlists, the playlist tabs of @@ -5142,6 +5181,9 @@ static enum scan_step task_manual_scan_iterate_step(retro_task_t *task, manual_scan->status = MANUAL_SCAN_ITERATE_CONTENT; else { + /* An m3u is counted in the m3u pass. */ + if (!rm3u_is_m3u_filestream(content_path)) + manual_scan->result.unmatched++; manual_scan->status = DATABASE_SCAN_ITERATE_NEXT; } break; @@ -5256,6 +5298,8 @@ static enum scan_step task_manual_scan_iterate_step(retro_task_t *task, label, sizeof(label))) { label[0] = '\0'; + if (!rm3u_is_m3u_filestream(content_path)) + manual_scan->result.unmatched++; #ifdef DEBUG RARCH_DBG("[Scanner] Rejecting item: %s\n",content_path); #endif @@ -5551,7 +5595,7 @@ static bool task_manual_content_scan_finder(retro_task_t *task, void *user_data) bool task_push_manual_content_scan( bool do_menu_refresh, - retro_task_callback_t user_cb) + retro_task_callback_t user_cb, void *user_data) { size_t _len; task_finder_data_t find_data; @@ -5675,6 +5719,7 @@ bool task_push_manual_content_scan( #endif manual_scan->user_cb = user_cb; + manual_scan->user_data = user_data; task->callback = cb_task_manual_content_scan; task->cleanup = task_manual_content_scan_free; task->flags |= RETRO_TASK_FLG_ALTERNATIVE_LOOK; diff --git a/tasks/tasks_internal.h b/tasks/tasks_internal.h index 4d116cdf394c..d448f8aaaf2e 100644 --- a/tasks/tasks_internal.h +++ b/tasks/tasks_internal.h @@ -336,16 +336,28 @@ bool task_push_image_load_set(const char *const *paths, * task. Safe from a task_queue_find predicate. */ bool task_image_set_tag(retro_task_t *task, uint64_t *tag); +/* A content scan's outcome, handed to its callback as task_data and + * valid only during the call. Counts cover every playlist the scan + * writes. */ +typedef struct +{ + unsigned added; + unsigned duplicates; + unsigned unmatched; + bool failed; +} task_dbscan_result_t; + #ifdef HAVE_LIBRETRODB /* Scans @fullpath, a directory or a single file, against the content * databases; the database and playlist directories come from the - * settings. */ -bool task_push_dbscan(const char *fullpath, retro_task_callback_t cb); + * settings. @cb gets @user_data. */ +bool task_push_dbscan(const char *fullpath, retro_task_callback_t cb, + void *user_data); #endif bool task_push_manual_content_scan( bool do_menu_refresh, - retro_task_callback_t user_cb); + retro_task_callback_t user_cb, void *user_data); #ifdef HAVE_OVERLAY bool task_push_overlay_load_default( diff --git a/ui/companion/companion_core.c b/ui/companion/companion_core.c index 3a52e3868f6a..db02845d7005 100644 --- a/ui/companion/companion_core.c +++ b/ui/companion/companion_core.c @@ -81,8 +81,9 @@ static void companion_core_download_orphan(companion_core_t *core); #endif #ifdef HAVE_LIBRETRODB -/* task_push_dbscan() carries no user data, so the requesting core is - * kept here. Only one desktop companion is active at a time. */ +/* The requesting core, kept here rather than passed as the scan's + * user data so that freeing the core clears it. Only one desktop + * companion is active at a time. */ static companion_core_t *companion_core_scan_owner = NULL; #endif @@ -1855,7 +1856,7 @@ bool companion_core_request_scan(companion_core_t *core, const char *path) return false; companion_core_scan_owner = core; - return task_push_dbscan(path, companion_core_scan_finished); + return task_push_dbscan(path, companion_core_scan_finished, NULL); #else (void)core; (void)path; diff --git a/ui/companion/test/companion_core_stubs.c b/ui/companion/test/companion_core_stubs.c index d4c651c954a4..3c62adaf6e68 100644 --- a/ui/companion/test/companion_core_stubs.c +++ b/ui/companion/test/companion_core_stubs.c @@ -209,9 +209,11 @@ bool task_push_load_content_with_current_core_from_companion_ui( bool task_push_start_current_core(content_ctx_info_t *ci) { (void)ci; stub_calls_start_core++; return true; } -bool task_push_dbscan(const char *path, retro_task_callback_t cb) +bool task_push_dbscan(const char *path, retro_task_callback_t cb, + void *user_data) { (void)cb; + (void)user_data; stub_calls_dbscan++; strlcpy(stub_last_scan_dir, path ? path : "", sizeof(stub_last_scan_dir)); return true; diff --git a/ui/drivers/ui_cocoatouch.m b/ui/drivers/ui_cocoatouch.m index 203b11560b89..ead1f899e36c 100644 --- a/ui/drivers/ui_cocoatouch.m +++ b/ui/drivers/ui_cocoatouch.m @@ -1507,7 +1507,7 @@ -(BOOL)application:(UIApplication *)app openURL:(NSURL *)url options:(NSDictiona sel_registerName("stopAccessingSecurityScopedResource")); } } - task_push_dbscan(fullpath, NULL); + task_push_dbscan(fullpath, NULL, NULL); return true; } From 6c6fac8db0d253cf58f0ed8f95bb05991c0ad560 Mon Sep 17 00:00:00 2001 From: davey Date: Fri, 30 Oct 2026 15:50:12 -0700 Subject: [PATCH 03/21] scan: say in the result when a scan failed A scan that ends before it runs (bad setup, no valid content, a DAT that will not load) or ends with a task error now sets the result's failed flag. A database folder that is missing or unset is treated as an empty one, so it reports the same "no database" error instead of finishing as if it had scanned. --- samples/tasks/database/database_scan_test.c | 86 +++++++++++++++++++++ tasks/task_database.c | 12 +++ 2 files changed, 98 insertions(+) diff --git a/samples/tasks/database/database_scan_test.c b/samples/tasks/database/database_scan_test.c index 77e5187434cb..6ae3f896c3b5 100644 --- a/samples/tasks/database/database_scan_test.c +++ b/samples/tasks/database/database_scan_test.c @@ -1891,6 +1891,7 @@ static bool scan_completed = false; static retro_time_t cb_usec; /* What the last run_scan() was handed: its user data is this. */ static task_dbscan_result_t scan_result; +static bool scan_err; static void scan_cb(retro_task_t *task, void *task_data, void *user_data, const char *err) @@ -1898,6 +1899,7 @@ static void scan_cb(retro_task_t *task, void *task_data, (void)task; if (err && *err) fprintf(stderr, " scan reported an error: %s\n", err); + scan_err = (err != NULL); if (user_data && task_data) *(task_dbscan_result_t*)user_data = *(const task_dbscan_result_t*)task_data; @@ -1937,6 +1939,7 @@ static bool run_scan(const char *dir) scan_result.duplicates = 9999; scan_result.unmatched = 9999; scan_result.failed = true; + scan_err = false; if (!task_push_dbscan(dir, scan_cb, &scan_result)) return false; @@ -2641,6 +2644,87 @@ static void use_db(const char *db_dir) sizeof(config_get_ptr()->paths.path_content_database)); } +/* A scan with no database to look in, or a DAT it cannot load, says + * it failed. */ +static void failed_lane(const char *root, const char *pl_dir, + const char *db_dir, uint32_t crc_a, uint32_t sz_a) +{ + char *dir = (char*)malloc(512); + char *nodb = (char*)malloc(512); + char *res_pl = (char*)malloc(512); + char *p = (char*)malloc(1024); + FILE *f; + bool refused; + + if (!dir || !nodb || !res_pl || !p) + { + check(0, "failed lane", "out of memory"); + goto end; + } + + sprintf(dir, "%s/scan_failed", root); + sprintf(nodb, "%s/scan_failed_nodb", root); + sprintf(res_pl, "%s/scan_failed_pl", root); + path_mkdir(dir); + path_mkdir(res_pl); + + sprintf(p, "%s/01_alpha.bin", dir); + if (!write_content(p, crc_a, sz_a)) + goto fixture; + sprintf(p, "%s/scan_failed.dat", root); + if (!(f = fopen(p, "w"))) + goto fixture; + fputs("\n" + "\n", f); + fclose(f); + + strlcpy(config_get_ptr()->paths.directory_playlist, res_pl, + sizeof(config_get_ptr()->paths.directory_playlist)); + + use_db(nodb); + run_scan(dir); + check_result("a missing database folder fails the scan", 0, 0, 1, true); + check(scan_err, "and reports it as an empty folder is", + scan_err ? "error" : "no error"); + use_db(""); + run_scan(dir); + check_result("as does no database folder set", 0, 0, 1, true); + use_db(db_dir); + check(scan_err, "and reports it too", + scan_err ? "error" : "no error"); + + /* Not a rescan of the folder the last scan set. */ + refused = !run_scan("/"); + check(refused, "a folder the scan settings reject is refused", + refused ? "refused" : "scanned"); + + manual_content_scan_set_menu_scan_method( + MANUAL_CONTENT_SCAN_METHOD_CUSTOM); + manual_content_scan_set_menu_scan_use_db( + MANUAL_CONTENT_SCAN_USE_DB_DAT_STRICT); + strlcpy(manual_content_scan_get_dat_file_path_ptr(), p, + manual_content_scan_get_dat_file_path_size()); + run_scan(dir); + check_result("a DAT that does not load fails the scan", 0, 0, 0, true); + + manual_content_scan_get_dat_file_path_ptr()[0] = '\0'; + manual_content_scan_set_menu_scan_use_db( + MANUAL_CONTENT_SCAN_USE_DB_STRICT); + manual_content_scan_set_menu_scan_method( + MANUAL_CONTENT_SCAN_METHOD_AUTOMATIC); + strlcpy(config_get_ptr()->paths.directory_playlist, pl_dir, + sizeof(config_get_ptr()->paths.directory_playlist)); + goto end; + +fixture: + check(0, "fixture", "could not write the failed lane"); +end: + free(dir); + free(nodb); + free(res_pl); + free(p); +} + static bool write_reader_db(const char *db_dir) { char p[1024]; @@ -8778,6 +8862,7 @@ int main(int argc, char **argv) "empty folder reported as invalid content", !scan_completed ? "timed out" : invalid_content_pushed ? "message pushed" : "no message"); + check_result("and fails the scan", 0, 0, 0, true); } /* A sheet that names itself is still the entry being scanned, so @@ -8943,6 +9028,7 @@ int main(int argc, char **argv) walk_lane(root, pl_dir, db_dir); single_lane(root, pl_dir, crc_a, sz_a); counts_lane(root, pl_dir, crc_a, sz_a, crc_d, sz_d); + failed_lane(root, pl_dir, db_dir, crc_a, sz_a); { char reader_db[512]; diff --git a/tasks/task_database.c b/tasks/task_database.c index 40807789973f..cbffe342303e 100644 --- a/tasks/task_database.c +++ b/tasks/task_database.c @@ -4431,8 +4431,12 @@ static void cb_task_manual_content_scan( * documented as menu-only and the sample in samples/tasks/database * hangs on exactly this. */ if (manual_scan && manual_scan->user_cb) + { + if (err) + manual_scan->result.failed = true; manual_scan->user_cb(task, &manual_scan->result, manual_scan->user_data, err); + } #if defined(HAVE_MENU) /* When creating playlists, the playlist tabs of @@ -4618,6 +4622,11 @@ static bool manual_scan_begin_setup(retro_task_t *task, } } + /* A missing or unset folder holds no database, as an empty + * one does: the end step reports it. */ + if (!dbstate->list && !(dbstate->list = string_list_new())) + return true; + RARCH_LOG("[Scanner] %s\"%s\"...\n", msg_hash_to_str(MSG_MANUAL_CONTENT_SCAN_START), manual_scan->content_database_path); if (manual_scan->task_config->cli_scan_output) printf("%s\"%s\"...\n", msg_hash_to_str(MSG_MANUAL_CONTENT_SCAN_START), manual_scan->content_database_path); @@ -5460,7 +5469,10 @@ static void task_manual_content_scan_handler(retro_task_t *task) case MANUAL_SCAN_BEGIN_DAT_LOAD: case MANUAL_SCAN_BEGIN_PLAYLIST: if (task_manual_scan_begin_tick(task, manual_scan)) + { + manual_scan->result.failed = true; goto task_finished; + } break; case MANUAL_SCAN_ITERATE_M3U: { From 525ec967e51d3c5b963d9083a22e1be4724b277d Mon Sep 17 00:00:00 2001 From: davey Date: Fri, 30 Oct 2026 16:05:27 -0700 Subject: [PATCH 04/21] scan: count the files a scan could not read A file whose scan fails (an unreadable file, a failed lookup) was in no count. The result now carries an errors count for them. They do not fail the scan. --- samples/tasks/database/database_scan_test.c | 90 +++++++++++++++++---- tasks/task_database.c | 19 ++++- tasks/tasks_internal.h | 3 +- 3 files changed, 91 insertions(+), 21 deletions(-) diff --git a/samples/tasks/database/database_scan_test.c b/samples/tasks/database/database_scan_test.c index 6ae3f896c3b5..5e684ffdb0ec 100644 --- a/samples/tasks/database/database_scan_test.c +++ b/samples/tasks/database/database_scan_test.c @@ -493,7 +493,8 @@ bool __wrap_task_queue_is_threaded(void) * counted by who makes them, a reader's on the main thread apart, and * those of begun_size bytes as each begins (reader_begun those a * reader begins); so are reads begun once cb_fired is set. A reader's - * read of a stream of fail_size bytes fails, counted in reader_fails. + * read of a stream of fail_size bytes fails, counted in reader_fails; + * every read of a stream of err_size bytes fails, the scan's own too. * The scan's first quantum of a stream of hold_size bytes waits, at * most HOLD_USEC, until hold_want reads of begun_size bytes have begun, * a slow one if slow_size is set and gate_n readers are in the gate @@ -518,6 +519,7 @@ static retro_atomic_int_t cb_fired; static retro_atomic_int_t reads_after_cb; static uint32_t fail_size; static retro_atomic_int_t reader_fails; +static uint32_t err_size; static uint32_t hold_size; static int hold_want; static int gate_n; @@ -604,6 +606,8 @@ int64_t __wrap_intfstream_crc_step(intfstream_internal_t *intf, #endif } } + if (err_size && size == (int64_t)err_size) + return -1; if (fail_size && size == (int64_t)fail_size && !scan) { retro_atomic_fetch_add_int(&reader_fails, 1); @@ -1938,6 +1942,7 @@ static bool run_scan(const char *dir) scan_result.added = 9999; scan_result.duplicates = 9999; scan_result.unmatched = 9999; + scan_result.errors = 9999; scan_result.failed = true; scan_err = false; if (!task_push_dbscan(dir, scan_cb, &scan_result)) @@ -2321,17 +2326,18 @@ static void single_lane(const char *root, const char *pl_dir, /* Checks the result the last run_scan() was handed. */ static void check_result(const char *what, unsigned added, - unsigned duplicates, unsigned unmatched, bool failed) + unsigned duplicates, unsigned unmatched, unsigned errors, bool failed) { char detail[128]; - sprintf(detail, "%u added, %u duplicates, %u unmatched%s", + sprintf(detail, "%u added, %u duplicates, %u unmatched, %u errors%s", scan_result.added, scan_result.duplicates, scan_result.unmatched, - scan_result.failed ? ", failed" : ""); + scan_result.errors, scan_result.failed ? ", failed" : ""); check( scan_completed && scan_result.added == added && scan_result.duplicates == duplicates && scan_result.unmatched == unmatched + && scan_result.errors == errors && scan_result.failed == failed, what, scan_completed ? detail : "refused or timed out"); } @@ -2435,9 +2441,9 @@ static void counts_lane(const char *root, const char *pl_dir, run_scan(dir); check_result("a scan counts what it adds and what matches nothing", - 2, 0, 2, false); + 2, 0, 2, 0, false); run_scan(dir); - check_result("a rescan counts what it already has", 0, 2, 2, false); + check_result("a rescan counts what it already has", 0, 2, 2, 0, false); delete_playlists(res_pl); manual_content_scan_set_menu_scan_method( @@ -2445,7 +2451,7 @@ static void counts_lane(const char *root, const char *pl_dir, manual_content_scan_set_menu_scan_use_db( MANUAL_CONTENT_SCAN_USE_DB_LOOSE); run_scan(dir); - check_result("a loose scan adds what matches nothing", 4, 0, 0, false); + check_result("a loose scan adds what matches nothing", 4, 0, 0, 0, false); delete_playlists(res_pl); manual_content_scan_set_menu_scan_use_db( @@ -2453,7 +2459,7 @@ static void counts_lane(const char *root, const char *pl_dir, strlcpy(manual_content_scan_get_dat_file_path_ptr(), p, manual_content_scan_get_dat_file_path_size()); run_scan(dat_dir); - check_result("a DAT scan counts what the DAT leaves out", 1, 0, 1, false); + check_result("a DAT scan counts what the DAT leaves out", 1, 0, 1, 0, false); manual_content_scan_get_dat_file_path_ptr()[0] = '\0'; manual_content_scan_set_menu_scan_use_db( @@ -2464,9 +2470,9 @@ static void counts_lane(const char *root, const char *pl_dir, /* An m3u counts once, as the one entry its discs fold into. */ delete_playlists(res_pl); run_scan(m3u_dir); - check_result("an m3u is added, not also unmatched", 1, 0, 0, false); + check_result("an m3u is added, not also unmatched", 1, 0, 0, 0, false); run_scan(m3u_dir); - check_result("an m3u rescan is one duplicate", 0, 1, 0, false); + check_result("an m3u rescan is one duplicate", 0, 1, 0, 0, false); /* One playlist, so the loose scan's own entry for the m3u and the * one its discs fold into meet there. */ @@ -2479,9 +2485,9 @@ static void counts_lane(const char *root, const char *pl_dir, MANUAL_CONTENT_SCAN_SYSTEM_NAME_CONTENT_DIR, NULL); run_scan(m3u_dir); check_result("a loose m3u is added, not also a duplicate", - 1, 0, 0, false); + 1, 0, 0, 0, false); run_scan(m3u_dir); - check_result("a loose m3u rescan is one duplicate", 0, 1, 0, false); + check_result("a loose m3u rescan is one duplicate", 0, 1, 0, 0, false); manual_content_scan_set_menu_system_name( MANUAL_CONTENT_SCAN_SYSTEM_NAME_AUTO, NULL); manual_content_scan_set_menu_scan_use_db( @@ -2504,6 +2510,57 @@ static void counts_lane(const char *root, const char *pl_dir, free(d); } +#define ERR_SIZE 1536 + +/* A file the scan cannot read is counted as an error; the scan goes + * on and does not fail. */ +static void errors_lane(const char *root, const char *pl_dir, + uint32_t crc_a, uint32_t sz_a) +{ + static const char m3u[] = "Missing (Disc 1).bin\n"; + char dir[512], m3u_dir[512], res_pl[512], p[1024]; + FILE *f; + + sprintf(dir, "%s/scan_errors", root); + sprintf(m3u_dir, "%s/scan_errors_m3u", root); + sprintf(res_pl, "%s/scan_errors_pl", root); + path_mkdir(dir); + path_mkdir(m3u_dir); + path_mkdir(res_pl); + sprintf(p, "%s/01_alpha.bin", dir); + if (!write_content(p, crc_a, sz_a)) + goto fixture; + sprintf(p, "%s/02_unread.bin", dir); + if (!write_content(p, 0x0E770001u, ERR_SIZE)) + goto fixture; + + delete_playlists(res_pl); + strlcpy(config_get_ptr()->paths.directory_playlist, res_pl, + sizeof(config_get_ptr()->paths.directory_playlist)); + err_size = ERR_SIZE; + run_scan(dir); + err_size = 0; + check_result("a file the scan cannot read is an error", 1, 0, 0, 1, + false); + + sprintf(p, "%s/Game.m3u", m3u_dir); + if (!(f = fopen(p, "wb"))) + goto fixture; + fputs(m3u, f); + fclose(f); + err_size = sizeof(m3u) - 1; + run_scan(m3u_dir); + err_size = 0; + check_result("an m3u the scan cannot read is an error, not also " + "unmatched", 0, 0, 0, 1, false); + strlcpy(config_get_ptr()->paths.directory_playlist, pl_dir, + sizeof(config_get_ptr()->paths.directory_playlist)); + return; + +fixture: + check(0, "fixture", "could not write the errors lane"); +} + /* The queue the next scans run on: threaded where threads are built, * else the regular one, so every build runs the same checks. */ static void use_queue(bool threaded) @@ -2683,12 +2740,12 @@ static void failed_lane(const char *root, const char *pl_dir, use_db(nodb); run_scan(dir); - check_result("a missing database folder fails the scan", 0, 0, 1, true); + check_result("a missing database folder fails the scan", 0, 0, 1, 0, true); check(scan_err, "and reports it as an empty folder is", scan_err ? "error" : "no error"); use_db(""); run_scan(dir); - check_result("as does no database folder set", 0, 0, 1, true); + check_result("as does no database folder set", 0, 0, 1, 0, true); use_db(db_dir); check(scan_err, "and reports it too", scan_err ? "error" : "no error"); @@ -2705,7 +2762,7 @@ static void failed_lane(const char *root, const char *pl_dir, strlcpy(manual_content_scan_get_dat_file_path_ptr(), p, manual_content_scan_get_dat_file_path_size()); run_scan(dir); - check_result("a DAT that does not load fails the scan", 0, 0, 0, true); + check_result("a DAT that does not load fails the scan", 0, 0, 0, 0, true); manual_content_scan_get_dat_file_path_ptr()[0] = '\0'; manual_content_scan_set_menu_scan_use_db( @@ -8862,7 +8919,7 @@ int main(int argc, char **argv) "empty folder reported as invalid content", !scan_completed ? "timed out" : invalid_content_pushed ? "message pushed" : "no message"); - check_result("and fails the scan", 0, 0, 0, true); + check_result("and fails the scan", 0, 0, 0, 0, true); } /* A sheet that names itself is still the entry being scanned, so @@ -9029,6 +9086,7 @@ int main(int argc, char **argv) single_lane(root, pl_dir, crc_a, sz_a); counts_lane(root, pl_dir, crc_a, sz_a, crc_d, sz_d); failed_lane(root, pl_dir, db_dir, crc_a, sz_a); + errors_lane(root, pl_dir, crc_a, sz_a); { char reader_db[512]; diff --git a/tasks/task_database.c b/tasks/task_database.c index cbffe342303e..5f38316966fe 100644 --- a/tasks/task_database.c +++ b/tasks/task_database.c @@ -921,7 +921,7 @@ static void remove_disc_indicators(char *title, size_t len) static void task_database_iterate_m3u( manual_scan_handle_t *_db, - const char *m3u_path) + const char *m3u_path, bool counted) { size_t i, j; bool found_match = false; @@ -939,7 +939,7 @@ static void task_database_iterate_m3u( if (!(m3u = rm3u_load_filestream(m3u_path))) { RARCH_ERR("[Scanner] Failed to open M3U file: \"%s\".\n", m3u_path); - if (!scan_results_find(&_db->scan_results, m3u_path)) + if (!counted && !scan_results_find(&_db->scan_results, m3u_path)) _db->result.unmatched++; return; } @@ -1053,7 +1053,7 @@ static void task_database_iterate_m3u( collapsed_title, first_matched_db); } } - else if (!own) + else if (!own && !counted) _db->result.unmatched++; /* Compact scan_results to remove NULL entries */ @@ -5205,6 +5205,15 @@ static enum scan_step task_manual_scan_iterate_step(retro_task_t *task, * same failing step every tick. Give up on this * file and go to the next one. */ RARCH_ERR("[Scanner] Scanning of content unexpectedly failed for \"%s\"\n", content_path); + /* Counted here, so not again in the m3u pass. */ + if ( manual_scan->m3u_list + && manual_scan->m3u_list->size + && string_is_equal(manual_scan->m3u_list->elems[ + manual_scan->m3u_list->size - 1].data, + content_path)) + manual_scan->m3u_list->elems[ + manual_scan->m3u_list->size - 1].attr.i = 1; + manual_scan->result.errors++; #ifdef SCAN_HAVE_READERS if (dbstate->crc_pass != DB_CRC_PASS_FILE) scan_readers_whole_ahead(manual_scan, 0); @@ -5501,7 +5510,9 @@ static void task_manual_content_scan_handler(retro_task_t *task) task_set_progress(task, (manual_scan->m3u_index * 100) / manual_scan->m3u_list->size); - task_database_iterate_m3u(manual_scan, m3u_path); + task_database_iterate_m3u(manual_scan, m3u_path, + manual_scan->m3u_list->elems[ + manual_scan->m3u_index].attr.i != 0); } /* Increment M3U file index */ diff --git a/tasks/tasks_internal.h b/tasks/tasks_internal.h index d448f8aaaf2e..d4aaa70e1ab2 100644 --- a/tasks/tasks_internal.h +++ b/tasks/tasks_internal.h @@ -338,12 +338,13 @@ bool task_image_set_tag(retro_task_t *task, uint64_t *tag); /* A content scan's outcome, handed to its callback as task_data and * valid only during the call. Counts cover every playlist the scan - * writes. */ + * writes; errors are files it could not read. */ typedef struct { unsigned added; unsigned duplicates; unsigned unmatched; + unsigned errors; bool failed; } task_dbscan_result_t; From a2f4f7e046b6932228d1cd16955958067bbed690 Mon Sep 17 00:00:00 2001 From: davey Date: Fri, 30 Oct 2026 16:09:02 -0700 Subject: [PATCH 05/21] scan: print --scan errors on stderr and totals on stdout A command-line scan now ends with a totals line on stdout, or with the reason it failed on stderr. A file it could not read is listed on stderr. The ++/==/?? marks are coloured only when their stream is a terminal, so pipes and redirected output get plain text; on Windows redirected output keeps its marks instead of losing them to WriteConsole. --- samples/tasks/database/database_scan_test.c | 172 +++++++++++++++++++- tasks/task_database.c | 112 ++++++++++--- 2 files changed, 263 insertions(+), 21 deletions(-) diff --git a/samples/tasks/database/database_scan_test.c b/samples/tasks/database/database_scan_test.c index 5e684ffdb0ec..76c3b212ce8d 100644 --- a/samples/tasks/database/database_scan_test.c +++ b/samples/tasks/database/database_scan_test.c @@ -35,6 +35,19 @@ #include #include #include +#ifdef _WIN32 +#include +#define cap_dup _dup +#define cap_dup2 _dup2 +#define cap_close _close +#define cap_fileno _fileno +#else +#include +#define cap_dup dup +#define cap_dup2 dup2 +#define cap_close close +#define cap_fileno fileno +#endif #include #include @@ -58,6 +71,7 @@ #include "../../../list_special.h" #include "../../../configuration.h" #include "../../../verbosity.h" +#include "../../../retroarch_types.h" #include "../../../libretro-db/libretrodb.h" #ifdef HAVE_SMBCLIENT #include "../../../libretro-common/vfs/vfs_implementation_smb.h" @@ -131,6 +145,8 @@ const char *msg_hash_to_str_us(enum msg_hash_enums msg) end_step_hold(); if (msg == MSG_MANUAL_CONTENT_SCAN_INVALID_CONTENT) return "invalid content"; + if (msg == MSG_SCANNING_NO_DATABASE) + return "no database"; return ""; } @@ -174,10 +190,13 @@ void runloop_msg_queue_push(const char *msg, size_t len, invalid_content_pushed = true; } +/* Whether the scan prints what --scan prints. */ +static bool cli_output; + bool retroarch_override_setting_is_set(unsigned enum_idx, void *data) { - (void)enum_idx; (void)data; - return false; + (void)data; + return cli_output && enum_idx == RARCH_OVERRIDE_SETTING_DATABASE_SCAN; } void ui_companion_driver_notify_refresh(void) @@ -2782,6 +2801,154 @@ static void failed_lane(const char *root, const char *pl_dir, free(p); } +/* stdout and stderr while capture_begin() has them in files. */ +static int cap_out = -1; +static int cap_err = -1; + +static void capture_end(void) +{ + fflush(stdout); + fflush(stderr); + if (cap_out >= 0) + { + cap_dup2(cap_out, cap_fileno(stdout)); + cap_close(cap_out); + cap_out = -1; + } + if (cap_err >= 0) + { + cap_dup2(cap_err, cap_fileno(stderr)); + cap_close(cap_err); + cap_err = -1; + } +} + +/* Sends stdout to @out and stderr to @err until capture_end(). The + * scan prints to the streams themselves, so their descriptors move. */ +static bool capture_begin(const char *out, const char *err) +{ + FILE *o = fopen(out, "w"); + FILE *e = fopen(err, "w"); + bool ok = false; + + fflush(stdout); + fflush(stderr); + if (o && e) + { + cap_out = cap_dup(cap_fileno(stdout)); + cap_err = cap_dup(cap_fileno(stderr)); + ok = cap_out >= 0 && cap_err >= 0 + && cap_dup2(cap_fileno(o), cap_fileno(stdout)) >= 0 + && cap_dup2(cap_fileno(e), cap_fileno(stderr)) >= 0; + } + if (o) + fclose(o); + if (e) + fclose(e); + if (!ok) + capture_end(); + return ok; +} + +/* Scans @dir with stdout in @out and stderr in @err; false if the + * streams could not be moved or the scan did not finish. */ +static bool capture_scan(const char *dir, const char *out, const char *err, + bool cli) +{ + bool captured; + + cli_output = cli; + if ((captured = capture_begin(out, err))) + { + run_scan(dir); + capture_end(); + } + cli_output = false; + check(captured && scan_completed, "--scan output to files", + !captured ? out : scan_completed ? "" : "timed out"); + return captured && scan_completed; +} + +/* What --scan prints, to files: the marks plain, the totals line on + * stdout, and on stderr a file the scan could not read and a scan + * that failed, which prints no totals. Without --scan, nothing. + * Rescans errors_lane()'s folder. */ +static void output_lane(const char *root, const char *pl_dir, + const char *db_dir) +{ + char *dir = (char*)malloc(512); + char *nodb = (char*)malloc(512); + char *empty = (char*)malloc(512); + char *res_pl = (char*)malloc(512); + char *out = (char*)malloc(512); + char *err = (char*)malloc(512); + char *want = (char*)malloc(1024); + + if (!dir || !nodb || !empty || !res_pl || !out || !err || !want) + { + check(0, "output lane", "out of memory"); + goto end; + } + + sprintf(dir, "%s/scan_errors", root); + sprintf(nodb, "%s/scan_output_nodb", root); + sprintf(empty, "%s/scan_output_empty", root); + sprintf(res_pl, "%s/scan_errors_pl", root); + sprintf(out, "%s/scan_output.out", root); + sprintf(err, "%s/scan_output.err", root); + path_mkdir(empty); + strlcpy(config_get_ptr()->paths.directory_playlist, res_pl, + sizeof(config_get_ptr()->paths.directory_playlist)); + + err_size = ERR_SIZE; + if (capture_scan(dir, out, err, true)) + { + /* No "\n" in the needles: Windows ends these lines "\r\n". */ + sprintf(want, "%s: 0 added, 1 duplicates, 0 unmatched, 1 errors", + dir); + check(file_contains(out, want), "--scan prints its totals on stdout", + want); + sprintf(want, " !! \"%s/02_unread.bin\"", dir); + check(file_contains(err, want), + "and a file it could not read on stderr", want); + check( file_contains(out, " == \"") + && !file_contains(out, "\x1B[") && !file_contains(err, "\x1B["), + "with no colour codes in a file", out); + } + + if (capture_scan(dir, out, err, false)) + check(path_get_size(out) == 0 && path_get_size(err) == 0, + "a scan without --scan prints nothing", out); + err_size = 0; + + use_db(nodb); + if (capture_scan(dir, out, err, true)) + { + sprintf(want, "%s: no database", dir); + check(file_contains(err, want) && !file_contains(out, " added, "), + "a failed scan says so on stderr, no totals", want); + } + use_db(db_dir); + + if (capture_scan(empty, out, err, true)) + { + sprintf(want, "%s: invalid content", empty); + check(file_contains(err, want) && !file_contains(out, " added, "), + "as does a scan that cannot start", want); + } + + strlcpy(config_get_ptr()->paths.directory_playlist, pl_dir, + sizeof(config_get_ptr()->paths.directory_playlist)); +end: + free(dir); + free(nodb); + free(empty); + free(res_pl); + free(out); + free(err); + free(want); +} + static bool write_reader_db(const char *db_dir) { char p[1024]; @@ -9087,6 +9254,7 @@ int main(int argc, char **argv) counts_lane(root, pl_dir, crc_a, sz_a, crc_d, sz_d); failed_lane(root, pl_dir, db_dir, crc_a, sz_a); errors_lane(root, pl_dir, crc_a, sz_a); + output_lane(root, pl_dir, db_dir); { char reader_db[512]; diff --git a/tasks/task_database.c b/tasks/task_database.c index 5f38316966fe..a40e3c8141b8 100644 --- a/tasks/task_database.c +++ b/tasks/task_database.c @@ -30,6 +30,10 @@ #include #include #include +#if !defined(_WIN32) && !defined(ORBIS) \ + && (defined(__unix__) || defined(__APPLE__)) +#include +#endif #ifdef HAVE_THREADS #include #include @@ -605,6 +609,8 @@ typedef struct manual_scan_handle void *user_data; /* The callback's task_data. */ task_dbscan_result_t result; + /* What a scan that could not start told the user. */ + const char *fail_msg; } manual_scan_handle_t; enum scan_verdict @@ -651,16 +657,43 @@ static const char *database_info_get_current_element_name( return handle->elems[ptr].data; } #endif -static void task_database_scan_console_output(const char *label, const char *db_name, bool add) +enum scan_console_mark +{ + SCAN_CONSOLE_ADDED = 0, + SCAN_CONSOLE_DUPLICATE, + SCAN_CONSOLE_UNMATCHED, + SCAN_CONSOLE_ERROR +}; + +/* Whether @stream is a terminal. Where that cannot be told, say yes. */ +static bool task_database_scan_console_is_tty(FILE *stream) +{ +#if defined(_WIN32) && !defined(_XBOX) && !defined(__WINRT__) + DWORD mode; + return GetConsoleMode(GetStdHandle((stream == stderr) + ? STD_ERROR_HANDLE : STD_OUTPUT_HANDLE), &mode) != 0; +#elif (defined(__unix__) || defined(__APPLE__)) && !defined(ORBIS) + /* The PS4 toolchain defines __unix__ but has no isatty(). */ + return isatty(fileno(stream)) != 0; +#else + return true; +#endif +} + +static void task_database_scan_console_output(FILE *stream, + const char *label, const char *db_name, enum scan_console_mark mark) { + static const char *const prefixes[] = { "++", "==", "??", "!!" }; char string[32]; - const char *prefix = (add) ? "++" : (db_name) ? "==" : "??"; + const char *prefix = prefixes[mark]; const char *no_color = getenv("NO_COLOR"); - bool color = (no_color && no_color[0] != '0') ? false : true; + bool color = (no_color && no_color[0] != '0') + ? false : task_database_scan_console_is_tty(stream); - /* Colorize prefix (add = green, dupe = yellow, not found = red) */ + /* Colorize prefix (add = green, dupe = yellow, not found or error = red) */ #ifdef _WIN32 - HANDLE con = GetStdHandle(STD_OUTPUT_HANDLE); + HANDLE con = GetStdHandle((stream == stderr) + ? STD_ERROR_HANDLE : STD_OUTPUT_HANDLE); if (color && con != INVALID_HANDLE_VALUE) { unsigned red = FOREGROUND_RED; @@ -670,9 +703,11 @@ static void task_database_scan_console_output(const char *label, const char *db_ size_t _len = strlcpy_lit(string, " ", sizeof(string)); _len += strlcpy(string + _len, prefix, sizeof(string) - _len); _len += strlcpy_lit(string + _len, " ", sizeof(string) - _len); - SetConsoleTextAttribute(con, (add) ? green : (db_name) ? yellow : red); + SetConsoleTextAttribute(con, (mark == SCAN_CONSOLE_ADDED) ? green + : (mark == SCAN_CONSOLE_DUPLICATE) ? yellow : red); WriteConsole(con, string, _len, NULL, NULL); SetConsoleTextAttribute(con, reset); + string[0] = '\0'; } #else if (color) @@ -682,15 +717,16 @@ static void task_database_scan_console_output(const char *label, const char *db_ const char *yellow = "\x1B[33m"; const char *reset = "\x1B[0m"; size_t _len = 0; - if (add) + if (mark == SCAN_CONSOLE_ADDED) _len += strlcpy(string + _len, green, sizeof(string) - _len); else - _len += strlcpy(string + _len, (db_name) ? yellow : red, sizeof(string) - _len); + _len += strlcpy(string + _len, + (mark == SCAN_CONSOLE_DUPLICATE) ? yellow : red, + sizeof(string) - _len); _len += strlcpy_lit(string + _len, " ", sizeof(string) - _len); _len += strlcpy(string + _len, prefix, sizeof(string) - _len); _len += strlcpy_lit(string + _len, " ", sizeof(string) - _len); strlcpy(string + _len, reset, sizeof(string) - _len); - fputs(string, stdout); } #endif else @@ -698,13 +734,14 @@ static void task_database_scan_console_output(const char *label, const char *db_ size_t _len = strlcpy_lit(string, " ", sizeof(string)); _len += strlcpy(string + _len, prefix, sizeof(string) - _len); strlcpy_lit(string + _len, " ", sizeof(string) - _len); - fputs(string, stdout); } + /* One call, so another thread's log line cannot split it. The + * Win32 colour prefix is written apart, before it. */ if (!db_name) - printf("\"%s\"\n", label); + fprintf(stream, "%s\"%s\"\n", string, label); else - printf("\"%s / %s\"\n", db_name, label); + fprintf(stream, "%s\"%s / %s\"\n", string, db_name, label); } #ifdef HAVE_LIBRETRODB static enum scan_verdict task_database_iterate_start(retro_task_t *task, @@ -2239,7 +2276,8 @@ static enum scan_verdict database_info_list_iterate_end_no_match( /* Reached end of database list, * CRC match probably didn't succeed. */ if (_db->task_config->cli_scan_output) - task_database_scan_console_output(path, NULL, false); + task_database_scan_console_output(stdout, path, NULL, + SCAN_CONSOLE_UNMATCHED); /* With its directory read, an archive's held hits were its * expansion. It still expands where a member stores no CRC, and @@ -4121,16 +4159,17 @@ static bool manual_scan_end_flush_tick( RARCH_LOG("[Scanner] Add \"%s / %s\".\n", db_name_noext, result->entry_label); if (manual_scan->task_config->cli_scan_output) - task_database_scan_console_output(result->entry_label, - db_name_noext, true); + task_database_scan_console_output(stdout, + result->entry_label, db_name_noext, SCAN_CONSOLE_ADDED); } /* Entry already exists - output duplicate indicator for CLI scans */ else if (manual_scan->flush_playlist) { manual_scan->result.duplicates++; if (manual_scan->task_config->cli_scan_output) - task_database_scan_console_output(result->entry_label, - db_name_noext, false); + task_database_scan_console_output(stdout, + result->entry_label, db_name_noext, + SCAN_CONSOLE_DUPLICATE); } manual_scan->flush_pos++; @@ -4653,6 +4692,7 @@ static bool manual_scan_begin_setup(retro_task_t *task, const char *_msg = msg_hash_to_str(MSG_MANUAL_CONTENT_SCAN_INVALID_CONTENT); runloop_msg_queue_push(_msg, strlen(_msg), 1, 100, true, NULL,\ MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_INFO); + manual_scan->fail_msg = _msg; return true; } } @@ -4694,6 +4734,7 @@ static bool manual_scan_begin_dir_list(retro_task_t *task, const char *_msg = msg_hash_to_str(MSG_MANUAL_CONTENT_SCAN_INVALID_CONTENT); runloop_msg_queue_push(_msg, strlen(_msg), 1, 100, true, NULL,\ MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_INFO); + manual_scan->fail_msg = _msg; } return true; } @@ -4712,6 +4753,7 @@ static bool manual_scan_begin_dir_list(retro_task_t *task, const char *_msg = msg_hash_to_str(MSG_MANUAL_CONTENT_SCAN_INVALID_CONTENT); runloop_msg_queue_push(_msg, strlen(_msg), 1, 100, true, NULL, MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_INFO); + manual_scan->fail_msg = _msg; return true; } @@ -4735,6 +4777,7 @@ static bool manual_scan_begin_dir_list(retro_task_t *task, const char *_msg = msg_hash_to_str(MSG_MANUAL_CONTENT_SCAN_INVALID_CONTENT); runloop_msg_queue_push(_msg, strlen(_msg), 1, 100, true, NULL,\ MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_INFO); + manual_scan->fail_msg = _msg; return true; } dir_list_sort(manual_scan->content_list, true); @@ -4876,6 +4919,7 @@ static bool manual_scan_begin_dat_load(retro_task_t *task, const char *_msg = msg_hash_to_str(MSG_MANUAL_CONTENT_SCAN_DAT_FILE_LOAD_ERROR); runloop_msg_queue_push(_msg, strlen(_msg), 1, 100, true, NULL, MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_INFO); + manual_scan->fail_msg = _msg; } return true; } @@ -5214,6 +5258,9 @@ static enum scan_step task_manual_scan_iterate_step(retro_task_t *task, manual_scan->m3u_list->elems[ manual_scan->m3u_list->size - 1].attr.i = 1; manual_scan->result.errors++; + if (manual_scan->task_config->cli_scan_output) + task_database_scan_console_output(stderr, + content_path, NULL, SCAN_CONSOLE_ERROR); #ifdef SCAN_HAVE_READERS if (dbstate->crc_pass != DB_CRC_PASS_FILE) scan_readers_whole_ahead(manual_scan, 0); @@ -5480,6 +5527,14 @@ static void task_manual_content_scan_handler(retro_task_t *task) if (task_manual_scan_begin_tick(task, manual_scan)) { manual_scan->result.failed = true; + if (manual_scan->task_config->cli_scan_output) + { + fprintf(stderr, "%s: %s\n", + manual_scan->task_config->content_dir, + manual_scan->fail_msg ? manual_scan->fail_msg + : msg_hash_to_str(MSG_MANUAL_CONTENT_SCAN_INVALID_CONFIG)); + fflush(stdout); + } goto task_finished; } break; @@ -5523,7 +5578,8 @@ static void task_manual_content_scan_handler(retro_task_t *task) break; case MANUAL_SCAN_END: { - const char *msg = NULL; + const char *msg = NULL; + bool no_database = false; /* Batch update all playlists with accumulated results, * spread across gathers under the shared window. */ @@ -5545,6 +5601,7 @@ static void task_manual_content_scan_handler(retro_task_t *task) { msg = msg_hash_to_str(MSG_SCANNING_NO_DATABASE); task_set_error(task, strdup(msg)); + no_database = true; } else { @@ -5561,7 +5618,24 @@ static void task_manual_content_scan_handler(retro_task_t *task) task_set_progress(task, 100); RARCH_LOG("[Scanner] %s\n", msg); if (manual_scan->task_config->cli_scan_output) - printf("%s\n", msg); + { + if (no_database) + fprintf(stderr, "%s: %s\n", + manual_scan->task_config->content_dir, msg); + else + { + printf("%s\n", msg); + printf("%s: %u added, %u duplicates, %u unmatched, %u errors\n", + manual_scan->task_config->content_dir, + manual_scan->result.added, + manual_scan->result.duplicates, + manual_scan->result.unmatched, + manual_scan->result.errors); + } + /* With --menu the process goes on: hand each scan's + * lines to a pipe or file as it ends. */ + fflush(stdout); + } RARCH_DBG("[Scanner] Scan settings were:\n"); RARCH_DBG("[Scanner] Content dir: \"%s\"\n",manual_scan->task_config->content_dir); From 84890107a429d0a17b1dd122c50ad40cefd3e5ce Mon Sep 17 00:00:00 2001 From: davey Date: Fri, 30 Oct 2026 19:05:03 -0700 Subject: [PATCH 06/21] scan: the manual scan finder skips a finished task A finished scan stays in the task queue while its callback runs, and the finder that refuses a second scan for the same playlist still matched it, so a scan pushed from that callback was refused. --- tasks/task_database.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tasks/task_database.c b/tasks/task_database.c index a40e3c8141b8..dcb81c4e9202 100644 --- a/tasks/task_database.c +++ b/tasks/task_database.c @@ -5684,6 +5684,10 @@ static bool task_manual_content_scan_finder(retro_task_t *task, void *user_data) return false; if (task->handler != task_manual_content_scan_handler) return false; + /* Still findable while its callback runs, which may push the + * next scan. */ + if (task_get_flags(task) & RETRO_TASK_FLG_FINISHED) + return false; if (!(manual_scan = (manual_scan_handle_t*)task->state)) return false; return string_is_equal( From 5fac6418139dc594a4f0595376906fea476c4c8e Mon Sep 17 00:00:00 2001 From: davey Date: Fri, 30 Oct 2026 16:13:10 -0700 Subject: [PATCH 07/21] retroarch: run every --scan, then exit with what the scans did Each --scan is collected while the command line is parsed and run in turn once it is done, so later options and further --scan flags are no longer dropped. A path is expanded and made absolute first; one that does not exist, or a scan that cannot start, is reported on stderr and the next path still runs. --scan with --cloudsync or --command is refused. Without --menu the exit status is 1 if any scan failed, else 0 if any added an entry, else 2, and a content path given with the scans is ignored, with a line on stderr. With --menu each finished scan starts the next, as a second scan is refused while one runs. --- retroarch.c | 225 ++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 192 insertions(+), 33 deletions(-) diff --git a/retroarch.c b/retroarch.c index ae7abe4af6bb..4cc58d0918b9 100644 --- a/retroarch.c +++ b/retroarch.c @@ -8260,6 +8260,166 @@ void handle_dbscan_finished(retro_task_t *task, void *task_data, void *user_data, const char *err); #endif +#ifdef HAVE_LIBRETRODB +/* @in with '~' expanded, made absolute if relative; NULL, with the + * reason on stderr, if nothing is there. Free the result. */ +static char *retroarch_cli_scan_path(const char *in) +{ + char *s = (char*)malloc(PATH_MAX_LENGTH); + + if (!s) + return NULL; + fill_pathname_expand_special(s, in, PATH_MAX_LENGTH); + /* "" would resolve to the working directory. Consoles resolve + * nothing, so a relative path stays as it is there. */ +#ifndef RARCH_CONSOLE + if ( *s && !path_is_absolute(s) + && !path_resolve_realpath(s, PATH_MAX_LENGTH, false)) + *s = '\0'; +#endif + if (!*s || !path_is_valid(s)) + { + fprintf(stderr, "%s: not found\n", in); + free(s); + return NULL; + } + return s; +} + +static void retroarch_cli_scan_done(retro_task_t *task, + void *task_data, void *user_data, const char *err) +{ + *(task_dbscan_result_t*)user_data = + *(const task_dbscan_result_t*)task_data; +} + +/* Scans @in to its end on this thread. */ +static void retroarch_cli_scan_one(const char *in, + task_dbscan_result_t *res) +{ + char *path = retroarch_cli_scan_path(in); + + memset(res, 0, sizeof(*res)); + res->failed = true; + if (!path) + return; + if (task_push_dbscan(path, retroarch_cli_scan_done, res)) + task_queue_wait(NULL, NULL); + else + fprintf(stderr, "%s: the scan could not start\n", path); + free(path); +} + +/* --scan with --menu: one scan at a time, as a second is refused + * while one runs; each pushes the next as it finishes. */ +typedef struct +{ + struct string_list *paths; + size_t next; +} retroarch_menu_scans_t; + +static void retroarch_menu_scan_next(retroarch_menu_scans_t *scans); + +static void retroarch_menu_scan_done(retro_task_t *task, + void *task_data, void *user_data, const char *err) +{ + retroarch_menu_scans_t *scans = (retroarch_menu_scans_t*)user_data; + +#ifdef HAVE_MENU + handle_dbscan_finished(task, task_data, NULL, err); +#endif + /* Cancelled, as on quit: start no more. */ + if (task_get_flags(task) & RETRO_TASK_FLG_CANCELLED) + scans->next = scans->paths->size; + retroarch_menu_scan_next(scans); +} + +/* Pushes the next path that starts; frees @scans after the last. */ +static void retroarch_menu_scan_next(retroarch_menu_scans_t *scans) +{ + while (scans->next < scans->paths->size) + { + bool pushed; + char *path = retroarch_cli_scan_path( + scans->paths->elems[scans->next++].data); + + if (!path) + continue; + pushed = task_push_dbscan(path, retroarch_menu_scan_done, scans); + if (!pushed) + fprintf(stderr, "%s: the scan could not start\n", path); + free(path); + if (pushed) + return; + } + string_list_free(scans->paths); + free(scans); +} + +/* Runs the --scan @paths, which it frees. With --menu the first is + * pushed and the rest follow it. Without, they are the whole run: + * each is scanned in turn, then the process exits 1 if any failed, + * else 0 if any added an entry, else 2. */ +static void retroarch_cli_scan(struct string_list *paths, bool menu, + const char *content) +{ + size_t i; + bool failed = false; + bool added = false; + settings_t *settings = config_get_ptr(); + int reinit_flags = DRIVERS_CMD_ALL & + ~(DRIVER_VIDEO_MASK | DRIVER_AUDIO_MASK | DRIVER_MICROPHONE_MASK | DRIVER_INPUT_MASK | DRIVER_MIDI_MASK); + + drivers_init(settings, reinit_flags, (enum driver_lifetime_flags)0, false); + retroarch_init_task_queue(); + + if (menu) + { + retroarch_menu_scans_t *scans = (retroarch_menu_scans_t*) + calloc(1, sizeof(*scans)); + + if (!scans) + { + string_list_free(paths); + return; + } + scans->paths = paths; + retroarch_menu_scan_next(scans); + return; + } + +#ifdef HAVE_THREADS + /* Threaded for this run whatever the setting: only there does the + * scan get its readers. */ + if (!task_queue_is_threaded()) + { + task_queue_set_threaded(); + task_queue_check(); + } +#endif + + if (content) + fprintf(stderr, "without --menu: ignoring \"%s\"\n", + content); + + for (i = 0; i < paths->size; i++) + { + task_dbscan_result_t res; + + retroarch_cli_scan_one(paths->elems[i].data, &res); + if (res.failed) + failed = true; + else if (res.added) + added = true; + } + string_list_free(paths); + + driver_uninit(DRIVERS_CMD_ALL, (enum driver_lifetime_flags)0); + task_queue_deinit(); + exit(failed ? 1 : added ? 0 : 2); +} +#endif + #ifdef HAVE_CLOUDSYNC /* The sync's verdict, where the waiting caller reads it. */ static void retroarch_cloud_sync_finished(retro_task_t *task, @@ -8313,8 +8473,9 @@ static bool retroarch_parse_input_and_config( access_state_t *access_st = access_state_get_ptr(); #endif #ifdef HAVE_LIBRETRODB - retro_task_callback_t cb_task_dbscan - = NULL; + struct string_list *scan_paths = NULL; + const char *scan_conflict = NULL; + bool cli_scan = false; #endif const struct option opts[] = { @@ -8572,12 +8733,21 @@ static bool retroarch_parse_input_and_config( break; case RA_OPT_DATABASE_SCAN: #ifdef HAVE_LIBRETRODB + cli_scan = true; verbosity_enable(); retroarch_override_setting_set(RARCH_OVERRIDE_SETTING_DATABASE_SCAN, NULL); #endif break; +#if defined(HAVE_LIBRETRODB) && defined(HAVE_NETWORKING) && defined(HAVE_NETWORK_CMD) && defined(HAVE_COMMAND) + case RA_OPT_COMMAND: + scan_conflict = "--command"; + break; +#endif case RA_OPT_CLOUDSYNC: #ifdef HAVE_CLOUDSYNC +#ifdef HAVE_LIBRETRODB + scan_conflict = "--cloudsync"; +#endif /* The sync's summary is the command's output. */ verbosity_enable(); retroarch_override_setting_set(RARCH_OVERRIDE_SETTING_VERBOSITY, NULL); @@ -8604,6 +8774,14 @@ static bool retroarch_parse_input_and_config( } } } +#ifdef HAVE_LIBRETRODB + /* Each of them is the whole run. */ + if (cli_scan && scan_conflict) + { + fprintf(stderr, "--scan cannot be combined with %s\n", scan_conflict); + exit(1); + } +#endif verbosity_enabled = verbosity_is_enabled(); /* Enable logging to file if verbosity and log-file arguments were passed. * RARCH_OVERRIDE_SETTING_LOG_TO_FILE is set by the RA_OPT_LOG_FILE case above @@ -9027,38 +9205,13 @@ static bool retroarch_parse_input_and_config( case RA_OPT_DATABASE_SCAN: #ifdef HAVE_LIBRETRODB { - settings_t *settings = config_get_ptr(); - int reinit_flags = DRIVERS_CMD_ALL & - ~(DRIVER_VIDEO_MASK | DRIVER_AUDIO_MASK | DRIVER_MICROPHONE_MASK | DRIVER_INPUT_MASK | DRIVER_MIDI_MASK); - - drivers_init(settings, reinit_flags, (enum driver_lifetime_flags)0, false); - retroarch_init_task_queue(); -#ifdef HAVE_THREADS - /* Threaded for this run whatever the setting: only - * there does the scan get its readers. */ - if (!explicit_menu && !task_queue_is_threaded()) - { - task_queue_set_threaded(); - task_queue_check(); - } -#endif - -#ifdef HAVE_MENU - if (explicit_menu) - cb_task_dbscan = handle_dbscan_finished; -#endif - - task_push_dbscan(optarg, cb_task_dbscan, NULL); + union string_list_elem_attr attr; - if (!explicit_menu) - { - /* Command-line scan with no menu: the scan is - * the whole run, so run the queue dry and exit. - * Nothing is drawn meanwhile. */ - task_queue_wait(NULL, NULL); - driver_uninit(DRIVERS_CMD_ALL, (enum driver_lifetime_flags)0); - exit(0); - } + attr.i = 0; + if (!scan_paths) + scan_paths = string_list_new(); + if (scan_paths) + string_list_append(scan_paths, optarg, attr); } #endif break; @@ -9105,6 +9258,12 @@ static bool retroarch_parse_input_and_config( } } +#ifdef HAVE_LIBRETRODB + if (scan_paths) + retroarch_cli_scan(scan_paths, explicit_menu, + (optind < argc) ? argv[optind] : NULL); +#endif + #ifdef HAVE_GIT_VERSION RARCH_LOG("RetroArch %s (Git %s)\n", PACKAGE_VERSION, retroarch_git_version); From 7c6f1167be74ac7229eaaca64d5ca2aaf532545e Mon Sep 17 00:00:00 2001 From: davey Date: Sun, 1 Nov 2026 00:23:13 -0700 Subject: [PATCH 08/21] manual_content_scan: clear the database name a refresh did not save A playlist with no saved database name left the name buffer uninitialised, and the refresh compared it against every installed database, so whatever was on the stack could pick a database the playlist never named. --- manual_content_scan.c | 2 + samples/tasks/database/database_scan_test.c | 94 +++++++++++++++++++++ 2 files changed, 96 insertions(+) diff --git a/manual_content_scan.c b/manual_content_scan.c index d5368dbc4212..a1b3a95d2213 100644 --- a/manual_content_scan.c +++ b/manual_content_scan.c @@ -729,6 +729,8 @@ enum manual_content_scan_playlist_refresh_status if (database_name_lpl && *database_name_lpl) fill_pathname(database_name,database_name_lpl,"",sizeof(database_name)); + else + database_name[0] = '\0'; /* Cannot happen, but would constitute a * 'system name' error */ if (!*system_name) diff --git a/samples/tasks/database/database_scan_test.c b/samples/tasks/database/database_scan_test.c index 76c3b212ce8d..ab4192e12f23 100644 --- a/samples/tasks/database/database_scan_test.c +++ b/samples/tasks/database/database_scan_test.c @@ -5091,6 +5091,99 @@ static void loose_held_lane(const char *root, const char *pl_dir, check(0, "fixture", "could not write the loose held set"); } +/* A refresh of a playlist that saved no database name leaves the + * database to the scan, whatever the refresh before it read. */ +static void refresh_db_name_lane(const char *root) +{ + static const char lpl[] = + "{\n" + " \"version\": \"1.5\",\n" + " \"scan_content_dir\": \"%s\",\n" + " \"scan_database_name\": \"%s\",\n" + " \"scan_search_recursively\": true,\n" + " \"scan_search_archives\": true,\n" + " \"scan_db_usage\": 0,\n" + " \"items\": []\n" + "}\n"; + static char text[2048]; + manual_content_scan_task_config_t *cfg = + (manual_content_scan_task_config_t*)calloc(1, sizeof(*cfg)); + playlist_config_t *pl_cfg = + (playlist_config_t*)calloc(1, sizeof(*pl_cfg)); + playlist_t *named = NULL; + playlist_t *plain = NULL; + char db[512], dir[512], pl[512], p[1024]; + db_rec_t rec; + bool ran; + + if (!cfg || !pl_cfg) + { + check(0, "refresh database name lane", "out of memory"); + goto end; + } + sprintf(db, "%s/refresh_name_db", root); + sprintf(dir, "%s/refresh_name", root); + sprintf(pl, "%s/refresh_name_pl", root); + path_mkdir(db); + path_mkdir(dir); + path_mkdir(pl); + set_rec(&rec, "Named Cart", 0x4E414D45u, 1024); + sprintf(p, "%s/Test Named.rdb", db); + if (!write_db_set(p, &rec, 1)) + goto fixture; + pl_cfg->capacity = COLLECTION_SIZE; + sprintf(p, "%s/Named.lpl", pl); + sprintf(text, lpl, dir, "Test Named.lpl"); + playlist_config_set_path(pl_cfg, p); + if ( !filestream_write_file(p, text, strlen(text)) + || !(named = playlist_init(pl_cfg))) + goto fixture; + sprintf(p, "%s/Plain.lpl", pl); + sprintf(text, lpl, dir, ""); + playlist_config_set_path(pl_cfg, p); + if ( !filestream_write_file(p, text, strlen(text)) + || !(plain = playlist_init(pl_cfg))) + goto fixture; + + /* Back to back: the second call's frame is where the first left + * its database name. */ + ran = manual_content_scan_set_menu_from_playlist(named, db, false) + == MANUAL_CONTENT_SCAN_PLAYLIST_REFRESH_OK + && manual_content_scan_set_menu_from_playlist(plain, db, false) + == MANUAL_CONTENT_SCAN_PLAYLIST_REFRESH_OK + && manual_content_scan_get_task_config(cfg, pl); + check(ran && !*cfg->database_name, + "refresh, no saved database: the scan picks", + !ran ? "refused" : *cfg->database_name ? cfg->database_name + : "automatic"); + ran = manual_content_scan_set_menu_from_playlist(named, db, false) + == MANUAL_CONTENT_SCAN_PLAYLIST_REFRESH_OK + && manual_content_scan_get_task_config(cfg, pl); + check(ran && !strcmp(cfg->database_name, "Test Named.lpl"), + "refresh, a saved database: that one", + ran ? cfg->database_name : "refused"); + goto restore; + +fixture: + check(0, "fixture", "could not write the refresh playlists"); +restore: + *manual_content_scan_get_validate_entries_ptr() = false; + manual_content_scan_set_menu_scan_db_select( + MANUAL_CONTENT_SCAN_SELECT_DB_AUTO, NULL); + manual_content_scan_get_system_name_custom_ptr()[0] = '\0'; + manual_content_scan_set_menu_system_name( + MANUAL_CONTENT_SCAN_SYSTEM_NAME_AUTO, NULL); + manual_content_scan_set_menu_scan_method( + MANUAL_CONTENT_SCAN_METHOD_AUTOMATIC); +end: + if (named) + playlist_free(named); + if (plain) + playlist_free(plain); + free(pl_cfg); + free(cfg); +} + #ifdef HAVE_RRAR /* Rar archives take the archive passes: a first member's hit, in a * database whose core opens rar, names the archive; a later member's, @@ -9303,6 +9396,7 @@ int main(int argc, char **argv) folder_first_lane(root, pl_dir, false); loose_held_lane(root, pl_dir, true); loose_held_lane(root, pl_dir, false); + refresh_db_name_lane(root); #ifdef HAVE_RRAR rar_lane(root, pl_dir, true); rar_lane(root, pl_dir, false); From 3554adf82ade85021bc8d73fd10d12b744e115fe Mon Sep 17 00:00:00 2001 From: davey Date: Sun, 1 Nov 2026 00:25:59 -0700 Subject: [PATCH 09/21] scan: Skip Database References names the scan's one playlist The option was meant to put the playlist's name in each entry's db_name, even for a database match (#18641), for users who build their own thumbnail sets. When the scan picks the playlists, db_name also picks the playlist an entry goes to, so with the option on every match went to one playlist named after the content folder. It now applies only when the scan writes one playlist, and gives that playlist's name, a loose scan's unmatched entries included. With the target on , or Fully Automatic with the option left on from a Custom scan, matches go to their databases' playlists as with it off; the entries of a custom playlist look their thumbnails up under its name, not the folder's. --- samples/tasks/database/database_scan_test.c | 114 ++++++++++++++++++++ tasks/task_database.c | 15 ++- 2 files changed, 126 insertions(+), 3 deletions(-) diff --git a/samples/tasks/database/database_scan_test.c b/samples/tasks/database/database_scan_test.c index ab4192e12f23..f1726ad1b1d1 100644 --- a/samples/tasks/database/database_scan_test.c +++ b/samples/tasks/database/database_scan_test.c @@ -5184,6 +5184,118 @@ static void refresh_db_name_lane(const char *root) free(cfg); } +static unsigned file_count(const char *path, const char *needle); + +/* Skip Database References names a match after the playlist it is + * written to, and only when the scan has one playlist: left to the + * scan, a match goes to its database's playlist. */ +static void omit_ref_lane(const char *root, const char *pl_dir, + bool threaded) +{ + static char cart_lpl[1024], folder_lpl[1024], custom_lpl[1024]; + char db[512], dir[512], pl[512], p[1024], what[80]; + db_rec_t cart; + bool started; + const char *q = threaded ? "(threaded)" : "(regular)"; + + sprintf(db, "%s/omit_ref_db", root); + sprintf(dir, "%s/omit_ref", root); + sprintf(pl, "%s/omit_ref_pl", root); + path_mkdir(db); + path_mkdir(dir); + path_mkdir(pl); + sprintf(p, "%s/om.ct", dir); + if (!write_content(p, 0x0A1E0F01u, 1024)) + goto fixture; + set_rec(&cart, "Omit Cart", 0x0A1E0F01u, 1024); + sprintf(p, "%s/Test Cart Gate.rdb", db); + if (!write_db_set(p, &cart, 1)) + goto fixture; + + sprintf(cart_lpl, "%s/Test Cart Gate.lpl", pl); + sprintf(folder_lpl, "%s/omit_ref.lpl", pl); + sprintf(custom_lpl, "%s/Omit Custom.lpl", pl); + filestream_delete(cart_lpl); + filestream_delete(folder_lpl); + use_queue(threaded); + use_db(db); + strlcpy(config_get_ptr()->paths.directory_playlist, pl, + sizeof(config_get_ptr()->paths.directory_playlist)); + *manual_content_scan_get_omit_db_ref_ptr() = true; + + manual_content_scan_set_menu_scan_method( + MANUAL_CONTENT_SCAN_METHOD_CUSTOM); + started = run_scan(dir) && scan_completed; + sprintf(what, "omit ref, Automatic target %s", q); + check( started && file_contains(cart_lpl, "om.ct") + && file_contains(cart_lpl, "\"db_name\": \"Test Cart Gate.lpl\"") + && !path_is_valid(folder_lpl), what, "Test Cart Gate.lpl"); + + /* Fully Automatic keeps the value Custom set. */ + filestream_delete(cart_lpl); + filestream_delete(folder_lpl); + manual_content_scan_set_menu_scan_method( + MANUAL_CONTENT_SCAN_METHOD_AUTOMATIC); + started = run_scan(dir) && scan_completed; + sprintf(what, "omit ref, Fully Automatic %s", q); + check( started && file_contains(cart_lpl, "om.ct") + && file_contains(cart_lpl, "\"db_name\": \"Test Cart Gate.lpl\"") + && !path_is_valid(folder_lpl), what, "Test Cart Gate.lpl"); + + manual_content_scan_set_menu_scan_method( + MANUAL_CONTENT_SCAN_METHOD_CUSTOM); + strlcpy(manual_content_scan_get_system_name_custom_ptr(), + "Omit Custom", + manual_content_scan_get_system_name_custom_size()); + manual_content_scan_set_menu_system_name( + MANUAL_CONTENT_SCAN_SYSTEM_NAME_CUSTOM, NULL); + filestream_delete(custom_lpl); + started = run_scan(dir) && scan_completed; + sprintf(what, "omit ref, Custom target: its name %s", q); + check( started + && file_contains(custom_lpl, "\"db_name\": \"Omit Custom.lpl\""), + what, "Omit Custom.lpl"); + + /* A loose scan's non-match too, not the folder's name. */ + sprintf(p, "%s/om_none.ct", dir); + if (!write_content(p, 0x0A1E0F02u, 1024)) + goto fixture; + manual_content_scan_set_menu_scan_use_db( + MANUAL_CONTENT_SCAN_USE_DB_LOOSE); + filestream_delete(custom_lpl); + started = run_scan(dir) && scan_completed; + manual_content_scan_set_menu_scan_use_db( + MANUAL_CONTENT_SCAN_USE_DB_STRICT); + filestream_delete(p); + sprintf(what, "omit ref, Custom target, loose: a non-match %s", q); + check( started && file_contains(custom_lpl, "om_none.ct") + && file_contains(custom_lpl, "om.ct") + && file_count(custom_lpl, "\"db_name\": \"Omit Custom.lpl\"") == 2 + && !file_contains(custom_lpl, "\"db_name\": \"omit_ref.lpl\"") + && !file_contains(custom_lpl, "\"db_name\": \"Test Cart Gate"), + what, custom_lpl); + + *manual_content_scan_get_omit_db_ref_ptr() = false; + filestream_delete(custom_lpl); + started = run_scan(dir) && scan_completed; + sprintf(what, "omit ref off: the database's name %s", q); + check( started + && file_contains(custom_lpl, "\"db_name\": \"Test Cart Gate.lpl\""), + what, "Test Cart Gate.lpl"); + + manual_content_scan_get_system_name_custom_ptr()[0] = '\0'; + manual_content_scan_set_menu_system_name( + MANUAL_CONTENT_SCAN_SYSTEM_NAME_AUTO, NULL); + manual_content_scan_set_menu_scan_method( + MANUAL_CONTENT_SCAN_METHOD_AUTOMATIC); + strlcpy(config_get_ptr()->paths.directory_playlist, pl_dir, + sizeof(config_get_ptr()->paths.directory_playlist)); + return; + +fixture: + check(0, "fixture", "could not write the omit ref set"); +} + #ifdef HAVE_RRAR /* Rar archives take the archive passes: a first member's hit, in a * database whose core opens rar, names the archive; a later member's, @@ -9397,6 +9509,8 @@ int main(int argc, char **argv) loose_held_lane(root, pl_dir, true); loose_held_lane(root, pl_dir, false); refresh_db_name_lane(root); + omit_ref_lane(root, pl_dir, true); + omit_ref_lane(root, pl_dir, false); #ifdef HAVE_RRAR rar_lane(root, pl_dir, true); rar_lane(root, pl_dir, false); diff --git a/tasks/task_database.c b/tasks/task_database.c index dcb81c4e9202..fe8331690f7b 100644 --- a/tasks/task_database.c +++ b/tasks/task_database.c @@ -2351,6 +2351,7 @@ static bool task_database_result_add(manual_scan_handle_t *_db, : (char*)malloc(db_crc_len); char* entry_path_str = (char*)malloc(str_len); char *hash = NULL; + const char *db_ref = db_playlist_base_str; const char *db_path = database_info_get_current_name(db_state); const char *entry_path = @@ -2434,11 +2435,14 @@ static bool task_database_result_add(manual_scan_handle_t *_db, && (hash = strchr(entry_path_str, '#'))) *hash = '\0'; + /* Without a fixed playlist, db_name picks the playlist. */ + if (_db->task_config->omit_db_reference && *_db->task_config->playlist_file) + db_ref = path_basename_nocompression(_db->task_config->playlist_file); + /* Accumulate result instead of immediately updating playlist */ - if (!scan_results_add(&_db->scan_results, entry_path_str, entry_lbl, db_crc, - _db->task_config->omit_db_reference ? _db->task_config->dat_file_path : db_playlist_base_str, - archive_name)) + if (!scan_results_add(&_db->scan_results, entry_path_str, entry_lbl, db_crc, + db_ref, archive_name)) RARCH_ERR("[Scanner] Failed to add result for: \"%s\".\n", entry_lbl); if (db_crc != db_crc_buf) @@ -5356,6 +5360,11 @@ static enum scan_step task_manual_scan_iterate_step(retro_task_t *task, *manual_scan->task_config->database_name ? manual_scan->task_config->database_name : manual_scan->task_config->dat_file_path; + /* Skip Database References, as for a match */ + if ( manual_scan->task_config->omit_db_reference + && *manual_scan->task_config->playlist_file) + db_name = path_basename_nocompression( + manual_scan->task_config->playlist_file); label[0] = '\0'; if (!manual_content_scan_get_playlist_content_label( content_path, manual_scan->dat_file, From 68d5f4e27eef86b2599a571cd1740746c75bfe47 Mon Sep 17 00:00:00 2001 From: davey Date: Sun, 1 Nov 2026 03:45:07 -0800 Subject: [PATCH 10/21] scan: name the scanned folder in the start line The "Scanning content" line, in the log and in --scan's output, has named the database folder since the manual scan and the database scan were merged (2fdf736d51); before, it named the path being scanned. It names the content folder again. --- samples/tasks/database/database_scan_test.c | 20 ++++++++++++++++++-- tasks/task_database.c | 4 ++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/samples/tasks/database/database_scan_test.c b/samples/tasks/database/database_scan_test.c index f1726ad1b1d1..a3c9c7f29b64 100644 --- a/samples/tasks/database/database_scan_test.c +++ b/samples/tasks/database/database_scan_test.c @@ -2882,9 +2882,11 @@ static void output_lane(const char *root, const char *pl_dir, char *res_pl = (char*)malloc(512); char *out = (char*)malloc(512); char *err = (char*)malloc(512); + char *log = (char*)malloc(512); char *want = (char*)malloc(1024); + bool logged; - if (!dir || !nodb || !empty || !res_pl || !out || !err || !want) + if (!dir || !nodb || !empty || !res_pl || !out || !err || !log || !want) { check(0, "output lane", "out of memory"); goto end; @@ -2896,6 +2898,7 @@ static void output_lane(const char *root, const char *pl_dir, sprintf(res_pl, "%s/scan_errors_pl", root); sprintf(out, "%s/scan_output.out", root); sprintf(err, "%s/scan_output.err", root); + sprintf(log, "%s/scan_output.log", root); path_mkdir(empty); strlcpy(config_get_ptr()->paths.directory_playlist, res_pl, sizeof(config_get_ptr()->paths.directory_playlist)); @@ -2914,11 +2917,23 @@ static void output_lane(const char *root, const char *pl_dir, check( file_contains(out, " == \"") && !file_contains(out, "\x1B[") && !file_contains(err, "\x1B["), "with no colour codes in a file", out); + sprintf(want, "\"%s\"...", dir); + check(file_contains(out, want), + "and starts with the folder it scans", want); } - if (capture_scan(dir, out, err, false)) + retro_main_log_file_init(log, false); + verbosity_enable(); + logged = capture_scan(dir, out, err, false); + verbosity_disable(); + retro_main_log_file_deinit(); + if (logged) + { check(path_get_size(out) == 0 && path_get_size(err) == 0, "a scan without --scan prints nothing", out); + sprintf(want, "\"%s\"...", dir); + check(file_contains(log, want), "but logs the folder it scans", want); + } err_size = 0; use_db(nodb); @@ -2946,6 +2961,7 @@ static void output_lane(const char *root, const char *pl_dir, free(res_pl); free(out); free(err); + free(log); free(want); } diff --git a/tasks/task_database.c b/tasks/task_database.c index fe8331690f7b..3a3567327fb6 100644 --- a/tasks/task_database.c +++ b/tasks/task_database.c @@ -4670,9 +4670,9 @@ static bool manual_scan_begin_setup(retro_task_t *task, if (!dbstate->list && !(dbstate->list = string_list_new())) return true; - RARCH_LOG("[Scanner] %s\"%s\"...\n", msg_hash_to_str(MSG_MANUAL_CONTENT_SCAN_START), manual_scan->content_database_path); + RARCH_LOG("[Scanner] %s\"%s\"...\n", msg_hash_to_str(MSG_MANUAL_CONTENT_SCAN_START), manual_scan->task_config->content_dir); if (manual_scan->task_config->cli_scan_output) - printf("%s\"%s\"...\n", msg_hash_to_str(MSG_MANUAL_CONTENT_SCAN_START), manual_scan->content_database_path); + printf("%s\"%s\"...\n", msg_hash_to_str(MSG_MANUAL_CONTENT_SCAN_START), manual_scan->task_config->content_dir); } } else From ca7b0bb13bbbb73fdb1f6205813773cf6c7157d1 Mon Sep 17 00:00:00 2001 From: davey Date: Sat, 31 Oct 2026 18:49:33 -0700 Subject: [PATCH 11/21] manual_content_scan: reset, scan overrides and their apply The command line's scan options need each scan to start from the defaults and to lay their values over them, or over a playlist's saved settings. Add a reset, a record of those options and a function that applies it through the existing setters: a core named by path, file name or display name, a database, a playlist, the database check or a DAT, extensions and the switches, stopping with a reason at the first that does not resolve. A core path is normalised before it is compared. A core's file name matches in the case given first, then ignoring case, on every platform; a display name matches exactly. Given another database, a playlist named after its database keeps that name as a custom one, so a refresh still writes the same playlist. The core setter's copy step becomes its own function, so a core found by path or file name can be set. Nothing calls the new functions yet. --- core_info.c | 2 +- core_info.h | 5 + manual_content_scan.c | 461 +++++++++++-- manual_content_scan.h | 53 ++ samples/tasks/database/database_scan_test.c | 681 ++++++++++++++++++++ 5 files changed, 1151 insertions(+), 51 deletions(-) diff --git a/core_info.c b/core_info.c index 1fb2138d5f8e..99c54d3b99c8 100644 --- a/core_info.c +++ b/core_info.c @@ -1648,7 +1648,7 @@ static bool core_info_path_is_standalone_exempt( return false; } -static size_t core_info_get_file_id(const char *core_filename, +size_t core_info_get_file_id(const char *core_filename, char *s, size_t len) { size_t _len; diff --git a/core_info.h b/core_info.h index fe3e2a3b7cb1..9740b6aca1de 100644 --- a/core_info.h +++ b/core_info.h @@ -200,6 +200,11 @@ bool core_info_list_update_missing_firmware(core_info_ctx_firmware_t *info); bool core_info_find(const char *core_path, core_info_t **core_info); +/* Writes the file id of 'core_filename' (no extension or + * platform suffix) to 's'; returns its length */ +size_t core_info_get_file_id(const char *core_filename, + char *s, size_t len); + bool core_info_load(const char *core_path); /* The extensions of every core that claims the database at diff --git a/manual_content_scan.c b/manual_content_scan.c index a1b3a95d2213..8b6d4e4145be 100644 --- a/manual_content_scan.c +++ b/manual_content_scan.c @@ -58,6 +58,20 @@ typedef struct bool validate_entries; } scan_settings_t; +#define MANUAL_CONTENT_SCAN_SETTINGS_DEFAULT { \ + MANUAL_CONTENT_SCAN_METHOD_AUTOMATIC, /* scan method */ \ + MANUAL_CONTENT_SCAN_SYSTEM_NAME_AUTO, /* system_name_type */ \ + MANUAL_CONTENT_SCAN_CORE_DETECT, /* core_type */ \ + MANUAL_CONTENT_SCAN_USE_DB_STRICT, \ + MANUAL_CONTENT_SCAN_SELECT_DB_AUTO, \ + true, /* search_recursively */ \ + true, /* search_archives */ \ + false, \ + false, /* filter_dat_content */ \ + false, /* overwrite_playlist */ \ + false /* validate_entries */ \ +} + /* TODO/FIXME - static public global variables */ /* Static settings object * > Provides easy access to settings parameters @@ -68,19 +82,7 @@ typedef struct * but this should not be an issue (i.e. regular configuration_settings * are not thread safe, but we only access them when pushing a * task, not in the task thread itself, so all is well) */ -static scan_settings_t scan_settings = { - MANUAL_CONTENT_SCAN_METHOD_AUTOMATIC, /* scan method */ - MANUAL_CONTENT_SCAN_SYSTEM_NAME_AUTO, /* system_name_type */ - MANUAL_CONTENT_SCAN_CORE_DETECT, /* core_type */ - MANUAL_CONTENT_SCAN_USE_DB_STRICT, - MANUAL_CONTENT_SCAN_SELECT_DB_AUTO, - true, /* search_recursively */ - true, /* search_archives */ - false, - false, /* filter_dat_content */ - false, /* overwrite_playlist */ - false /* validate_entries */ -}; +static scan_settings_t scan_settings = MANUAL_CONTENT_SCAN_SETTINGS_DEFAULT; /* The path and name buffers live outside the struct: three * non-zero defaults were dragging eleven kilobytes of zero buffers @@ -361,6 +363,23 @@ enum manual_content_scan_dat_file_path_status /* Menu setters */ +/* Sets every setting back to its default */ +void manual_content_scan_reset_settings(void) +{ + static const scan_settings_t d = MANUAL_CONTENT_SCAN_SETTINGS_DEFAULT; + + scan_settings = d; + scan_core_path[0] = '\0'; + scan_file_exts_core[0] = '\0'; + scan_file_exts_custom[0] = '\0'; + scan_dat_file_path[0] = '\0'; + scan_content_dir[0] = '\0'; + scan_system_name_content_dir[0] = '\0'; + scan_system_name_database[0] = '\0'; + scan_system_name_custom[0] = '\0'; + scan_core_name[0] = '\0'; +} + /* Sets content directory for next manual scan * operation. * Returns true if content directory is valid. */ @@ -555,6 +574,53 @@ bool manual_content_scan_set_menu_system_name( return false; } +/* Copies the core's path, name and extensions + * to the settings. + * Returns false if the core has no path. */ +static bool manual_content_scan_set_core(const core_info_t *core_info) +{ + /* Copy core path to settings struct */ + if (!core_info->path || !*core_info->path) + return false; + + strlcpy( + scan_core_path, + core_info->path, + sizeof(scan_core_path)); + + /* Copy core name to settings struct */ + strlcpy( + scan_core_name, + core_info->display_name, + sizeof(scan_core_name)); + + /* Copy supported extensions to settings + * struct, if required */ + if ( core_info->supported_extensions + && *core_info->supported_extensions) + { + strlcpy( + scan_file_exts_core, + core_info->supported_extensions, + sizeof(scan_file_exts_core)); + + /* Core info extensions are delimited by + * vertical bars. For internal consistency, + * replace them with spaces */ + string_replace_all_chars(scan_file_exts_core, '|', ' '); + + /* Apply standard scrubbing/clean-up + * (should not be required, but must handle the + * case where a core info file is incorrectly + * formatted) */ + manual_content_scan_scrub_file_exts(scan_file_exts_core); + } + else + scan_file_exts_core[0] = '\0'; + + return true; +} + /* Sets core name for the next manual scan * operation (+ core path and other associated * parameters). @@ -607,45 +673,9 @@ bool manual_content_scan_set_menu_core_name( /* Core has been found */ core_found = true; - /* Copy core path to settings struct */ - if (!core_info->path || !*core_info->path) + if (!manual_content_scan_set_core(core_info)) goto error; - strlcpy( - scan_core_path, - core_info->path, - sizeof(scan_core_path)); - - /* Copy core name to settings struct */ - strlcpy( - scan_core_name, - core_info->display_name, - sizeof(scan_core_name)); - - /* Copy supported extensions to settings - * struct, if required */ - if ( core_info->supported_extensions - && *core_info->supported_extensions) - { - strlcpy( - scan_file_exts_core, - core_info->supported_extensions, - sizeof(scan_file_exts_core)); - - /* Core info extensions are delimited by - * vertical bars. For internal consistency, - * replace them with spaces */ - string_replace_all_chars(scan_file_exts_core, '|', ' '); - - /* Apply standard scrubbing/clean-up - * (should not be required, but must handle the - * case where a core info file is incorrectly - * formatted) */ - manual_content_scan_scrub_file_exts(scan_file_exts_core); - } - else - scan_file_exts_core[0] = '\0'; - break; } } @@ -882,6 +912,337 @@ enum manual_content_scan_playlist_refresh_status return MANUAL_CONTENT_SCAN_PLAYLIST_REFRESH_OK; } +/* Whether @name is an installed database: the basename, + * without extension, of a path in @rdb_list */ +static bool manual_content_scan_rdb_list_has( + const struct string_list *rdb_list, const char *name) +{ + size_t i; + + if (!rdb_list) + return false; + + for (i = 0; i < rdb_list->size; i++) + { + char rdb_name[NAME_MAX_LENGTH]; + const char *rdb_path = rdb_list->elems[i].data; + if (!rdb_path || !*rdb_path) + continue; + fill_pathname(rdb_name, path_basename(rdb_path), "", + sizeof(rdb_name)); + if (string_is_equal(name, rdb_name)) + return true; + } + + return false; +} + +enum manual_content_scan_core_key +{ + MANUAL_CONTENT_SCAN_CORE_KEY_PATH = 0, + MANUAL_CONTENT_SCAN_CORE_KEY_FILE_ID_EXACT, + MANUAL_CONTENT_SCAN_CORE_KEY_FILE_ID, + MANUAL_CONTENT_SCAN_CORE_KEY_DISPLAY_NAME +}; + +/* Counts the cores whose @key is @s; *found gets the first */ +static unsigned manual_content_scan_match_cores( + core_info_list_t *core_info_list, + enum manual_content_scan_core_key key, const char *s, + const core_info_t **found) +{ + size_t i; + unsigned count = 0; +#ifdef _WIN32 + /* Either slash: core_info joins its folder and file with '/' */ + char *core_path = NULL; + + if ( key == MANUAL_CONTENT_SCAN_CORE_KEY_PATH + && !(core_path = (char*)malloc(PATH_MAX_LENGTH))) + return 0; +#endif + + for (i = 0; i < core_info_list->count; i++) + { + const core_info_t *core_info = core_info_get(core_info_list, i); + bool match = false; + + if (!core_info) + continue; + + switch (key) + { + case MANUAL_CONTENT_SCAN_CORE_KEY_PATH: +#ifdef _WIN32 + if (core_info->path) + { + strlcpy(core_path, core_info->path, PATH_MAX_LENGTH); + pathname_conform_slashes_to_os(core_path); + match = string_is_equal_case_insensitive(core_path, s); + } +#else + match = string_is_equal(core_info->path, s); +#endif + break; + case MANUAL_CONTENT_SCAN_CORE_KEY_FILE_ID_EXACT: + match = string_is_equal(core_info->core_file_id.str, s); + break; + case MANUAL_CONTENT_SCAN_CORE_KEY_FILE_ID: + match = string_is_equal_case_insensitive( + core_info->core_file_id.str, s); + break; + case MANUAL_CONTENT_SCAN_CORE_KEY_DISPLAY_NAME: + match = string_is_equal(core_info->display_name, s); + break; + } + + if (match && !count++) + *found = core_info; + } + +#ifdef _WIN32 + free(core_path); +#endif + return count; +} + +/* Finds the core @core names: a path, else a file name or + * file id, else a display name. The first of these that + * matches decides, and *count gets its matches. */ +static const core_info_t *manual_content_scan_find_core( + const char *core, unsigned *count) +{ + core_info_list_t *core_info_list = NULL; + const core_info_t *found = NULL; + char *s = NULL; + char *lower = NULL; + + *count = 0; + core_info_get_list(&core_info_list); + + if (!core_info_list || !(s = (char*)malloc(PATH_MAX_LENGTH))) + return NULL; + + /* Display names hold slashes too, so a path + * that matches no core falls through */ + if ( strchr(core, '/') +#ifdef _WIN32 + || strchr(core, '\\') +#endif + || *core == '~') + { + fill_pathname_expand_special(s, core, PATH_MAX_LENGTH); +#ifndef RARCH_CONSOLE + /* Folds "//" and ".." in an absolute path too */ + if ( *s + && !path_resolve_realpath(s, PATH_MAX_LENGTH, false) + && !path_is_absolute(s)) + *s = '\0'; +#endif +#ifdef _WIN32 + pathname_conform_slashes_to_os(s); +#endif + if (*s) + *count = manual_content_scan_match_cores(core_info_list, + MANUAL_CONTENT_SCAN_CORE_KEY_PATH, s, &found); + } + + /* Lower case, as core_info finds "_libretro" in lower case only */ + if (!*count && (lower = strdup(core))) + { + size_t i, _len; + string_to_lower(lower); + /* core_info's rule would cut a bare id's last "_" part */ + if (strstr(lower, "_libretro")) + _len = core_info_get_file_id(lower, s, PATH_MAX_LENGTH); + else + { + _len = fill_pathname(s, lower, "", PATH_MAX_LENGTH); + strlcpy(s + _len, "_libretro", PATH_MAX_LENGTH - _len); + } + /* The case typed, for an exact match first */ + for (i = 0; i < _len; i++) + if (s[i] == lower[i]) + s[i] = core[i]; + free(lower); + *count = manual_content_scan_match_cores(core_info_list, + MANUAL_CONTENT_SCAN_CORE_KEY_FILE_ID_EXACT, s, &found); + if (!*count) + *count = manual_content_scan_match_cores(core_info_list, + MANUAL_CONTENT_SCAN_CORE_KEY_FILE_ID, s, &found); + } + + if (!*count) + *count = manual_content_scan_match_cores(core_info_list, + MANUAL_CONTENT_SCAN_CORE_KEY_DISPLAY_NAME, core, &found); + + free(s); + return found; +} + +manual_content_scan_overrides_t *manual_content_scan_overrides_new(void) +{ + return (manual_content_scan_overrides_t*)calloc(1, + sizeof(manual_content_scan_overrides_t)); +} + +void manual_content_scan_overrides_free(manual_content_scan_overrides_t *ov) +{ + if (!ov) + return; + free(ov->playlist); + free(ov->database); + free(ov->dat); + free(ov->core); + free(ov->exts); + free(ov); +} + +/* Lays @ov over the current settings and sets the Custom + * method, stopping at the first override that does not + * resolve; NULL @ov changes nothing. + * > @rdb_list: paths of the installed databases, or NULL + * > @count: the cores @ov->core matched, when not NULL */ +enum manual_content_scan_override_status + manual_content_scan_apply_overrides( + const manual_content_scan_overrides_t *ov, + const struct string_list *rdb_list, unsigned *count) +{ + if (count) + *count = 0; + + if (!ov) + return MANUAL_CONTENT_SCAN_OVR_OK; + + if (ov->core) + { + unsigned matches = 0; + const core_info_t *core_info = manual_content_scan_find_core( + ov->core, &matches); + + if (count) + *count = matches; + if (matches > 1) + return MANUAL_CONTENT_SCAN_OVR_AMBIGUOUS_CORE; + if (!core_info || !manual_content_scan_set_core(core_info)) + return MANUAL_CONTENT_SCAN_OVR_NO_SUCH_CORE; + scan_settings.core_type = MANUAL_CONTENT_SCAN_CORE_SET; + } + + if (ov->database) + { + if (!manual_content_scan_rdb_list_has(rdb_list, ov->database)) + return MANUAL_CONTENT_SCAN_OVR_NO_SUCH_DATABASE; + /* A refreshed database playlist's name shares this + * buffer: it keeps that name as a custom one */ + if ( scan_settings.system_name_type + == MANUAL_CONTENT_SCAN_SYSTEM_NAME_DATABASE + && !string_is_equal(scan_system_name_database, ov->database)) + { + strlcpy(scan_system_name_custom, scan_system_name_database, + sizeof(scan_system_name_custom)); + manual_content_scan_set_menu_system_name( + MANUAL_CONTENT_SCAN_SYSTEM_NAME_CUSTOM, NULL); + } + manual_content_scan_set_menu_scan_db_select( + MANUAL_CONTENT_SCAN_SELECT_DB_SPECIFIC, ov->database); + } + else if (ov->flags & MANUAL_CONTENT_SCAN_OVR_FIRST_DB) + manual_content_scan_set_menu_scan_db_select( + MANUAL_CONTENT_SCAN_SELECT_DB_AUTO_FIRST_MATCH, NULL); + + /* A database playlist's name and the database to match + * share one buffer: next to another database, the + * playlist is custom */ + if (ov->playlist) + { + if ( manual_content_scan_rdb_list_has(rdb_list, ov->playlist) + && (!ov->database || string_is_equal(ov->database, ov->playlist))) + manual_content_scan_set_menu_system_name( + MANUAL_CONTENT_SCAN_SYSTEM_NAME_DATABASE, ov->playlist); + else + { + strlcpy(scan_system_name_custom, ov->playlist, + sizeof(scan_system_name_custom)); + manual_content_scan_scrub_system_name_custom(); + manual_content_scan_set_menu_system_name( + MANUAL_CONTENT_SCAN_SYSTEM_NAME_CUSTOM, NULL); + } + } + + if (ov->dat) + { + if ( (ov->flags & MANUAL_CONTENT_SCAN_OVR_MATCH) + && ov->match == MANUAL_CONTENT_SCAN_USE_DB_NONE) + return MANUAL_CONTENT_SCAN_OVR_DAT_WITHOUT_MATCH; + + fill_pathname_expand_special(scan_dat_file_path, ov->dat, + sizeof(scan_dat_file_path)); +#ifndef RARCH_CONSOLE + if ( *scan_dat_file_path && !path_is_absolute(scan_dat_file_path) + && !path_resolve_realpath(scan_dat_file_path, + sizeof(scan_dat_file_path), false)) + scan_dat_file_path[0] = '\0'; +#endif + switch (manual_content_scan_validate_dat_file_path()) + { + case MANUAL_CONTENT_SCAN_DAT_FILE_OK: + break; + case MANUAL_CONTENT_SCAN_DAT_FILE_TOO_LARGE: + return MANUAL_CONTENT_SCAN_OVR_DAT_TOO_LARGE; + default: + return MANUAL_CONTENT_SCAN_OVR_NOT_A_DAT; + } + + manual_content_scan_set_menu_scan_use_db( + ( (ov->flags & MANUAL_CONTENT_SCAN_OVR_MATCH) + && ov->match == MANUAL_CONTENT_SCAN_USE_DB_LOOSE) + ? MANUAL_CONTENT_SCAN_USE_DB_DAT_LOOSE + : MANUAL_CONTENT_SCAN_USE_DB_DAT_STRICT); + } + else if (ov->flags & MANUAL_CONTENT_SCAN_OVR_MATCH) + { + enum manual_content_scan_db_usage usage = ov->match; + + /* A refresh keeps its saved DAT unless the match is none */ + if ( scan_settings.db_usage == MANUAL_CONTENT_SCAN_USE_DB_DAT_STRICT + || scan_settings.db_usage == MANUAL_CONTENT_SCAN_USE_DB_DAT_LOOSE) + { + if (ov->match == MANUAL_CONTENT_SCAN_USE_DB_STRICT) + usage = MANUAL_CONTENT_SCAN_USE_DB_DAT_STRICT; + else if (ov->match == MANUAL_CONTENT_SCAN_USE_DB_LOOSE) + usage = MANUAL_CONTENT_SCAN_USE_DB_DAT_LOOSE; + else + scan_dat_file_path[0] = '\0'; + } + + manual_content_scan_set_menu_scan_use_db(usage); + } + + if (ov->exts) + { + strlcpy(scan_file_exts_custom, ov->exts, + sizeof(scan_file_exts_custom)); + manual_content_scan_scrub_file_exts_custom(); + } + + if (ov->flags & MANUAL_CONTENT_SCAN_OVR_NO_RECURSIVE) + scan_settings.search_recursively = false; + if (ov->flags & MANUAL_CONTENT_SCAN_OVR_NO_ARCHIVES) + scan_settings.search_archives = false; + if (ov->flags & MANUAL_CONTENT_SCAN_OVR_OVERWRITE) + scan_settings.overwrite_playlist = true; + if (ov->flags & MANUAL_CONTENT_SCAN_OVR_VALIDATE) + scan_settings.validate_entries = true; + if (ov->flags & MANUAL_CONTENT_SCAN_OVR_OMIT_DB_REF) + scan_settings.omit_db_ref = true; + + manual_content_scan_set_menu_scan_method( + MANUAL_CONTENT_SCAN_METHOD_CUSTOM); + + return MANUAL_CONTENT_SCAN_OVR_OK; +} + /* Menu getters */ /* Fetches content directory for next manual scan diff --git a/manual_content_scan.h b/manual_content_scan.h index 8b6177129314..1738e03f1172 100644 --- a/manual_content_scan.h +++ b/manual_content_scan.h @@ -104,6 +104,43 @@ enum manual_content_scan_playlist_refresh_status MANUAL_CONTENT_SCAN_PLAYLIST_REFRESH_DAT_FILE_TOO_LARGE }; +/* Defines all possible return values for + * manual_content_scan_apply_overrides() */ +enum manual_content_scan_override_status +{ + MANUAL_CONTENT_SCAN_OVR_OK = 0, + MANUAL_CONTENT_SCAN_OVR_NO_SUCH_CORE, + MANUAL_CONTENT_SCAN_OVR_AMBIGUOUS_CORE, + MANUAL_CONTENT_SCAN_OVR_NO_SUCH_DATABASE, + MANUAL_CONTENT_SCAN_OVR_NOT_A_DAT, + MANUAL_CONTENT_SCAN_OVR_DAT_TOO_LARGE, + MANUAL_CONTENT_SCAN_OVR_DAT_WITHOUT_MATCH +}; + +enum manual_content_scan_override_flags +{ + MANUAL_CONTENT_SCAN_OVR_MATCH = (1 << 0), /* 'match' is set */ + MANUAL_CONTENT_SCAN_OVR_FIRST_DB = (1 << 1), + MANUAL_CONTENT_SCAN_OVR_NO_RECURSIVE = (1 << 2), + MANUAL_CONTENT_SCAN_OVR_NO_ARCHIVES = (1 << 3), + MANUAL_CONTENT_SCAN_OVR_OVERWRITE = (1 << 4), + MANUAL_CONTENT_SCAN_OVR_VALIDATE = (1 << 5), + MANUAL_CONTENT_SCAN_OVR_OMIT_DB_REF = (1 << 6) +}; + +/* Settings laid over a scan's own; a NULL string or a clear + * flag leaves its setting as it is */ +typedef struct manual_content_scan_overrides +{ + char *playlist; /* --scan-playlist */ + char *database; /* --scan-database=NAME */ + char *dat; /* --scan-dat, as typed */ + char *core; /* --scan-core, as typed */ + char *exts; /* --scan-exts, space separated */ + enum manual_content_scan_db_usage match; /* STRICT, LOOSE or NONE */ + uint16_t flags; /* MANUAL_CONTENT_SCAN_OVR_* */ +} manual_content_scan_overrides_t; + /* Holds all configuration parameters required * for a manual content scan task */ typedef struct @@ -223,6 +260,9 @@ enum manual_content_scan_dat_file_path_status /* Menu setters */ +/* Sets every setting back to its default */ +void manual_content_scan_reset_settings(void); + /* Sets content directory for next manual scan * operation. * Returns true if content directory is valid. */ @@ -269,6 +309,19 @@ enum manual_content_scan_playlist_refresh_status manual_content_scan_set_menu_from_playlist(playlist_t *playlist, const char *path_content_database, bool show_hidden_files); +manual_content_scan_overrides_t *manual_content_scan_overrides_new(void); +void manual_content_scan_overrides_free(manual_content_scan_overrides_t *ov); + +/* Lays @ov over the current settings and sets the Custom + * method, stopping at the first override that does not + * resolve; NULL @ov changes nothing. + * > @rdb_list: paths of the installed databases, or NULL + * > @count: the cores @ov->core matched, when not NULL */ +enum manual_content_scan_override_status + manual_content_scan_apply_overrides( + const manual_content_scan_overrides_t *ov, + const struct string_list *rdb_list, unsigned *count); + /* Menu getters */ /* Fetches content directory for next manual scan diff --git a/samples/tasks/database/database_scan_test.c b/samples/tasks/database/database_scan_test.c index a3c9c7f29b64..e106487fd993 100644 --- a/samples/tasks/database/database_scan_test.c +++ b/samples/tasks/database/database_scan_test.c @@ -8396,6 +8396,682 @@ static void share_gate_lane(const char *root, const char *db_dir) #endif } +/* Cores for core_lookup_lane(): two files with one file id, two cores + * with one display name, a display name with a slash, a file id + * with an underscore and two file names that differ only in case. */ +static bool write_lookup_cores(const char *info_dir, const char *core_dir) +{ + static const char *const info[][3] = { + { "dup", "Dup Test", "dupx" }, + { "same1", "Same Name Test", "samex" }, + { "same2", "Same Name Test", "samex" }, + { "slash", "Slash / Test (Lookup)", "slashx" }, + { "two_part", "Two Part Test", "twox" } + }; + static const char *const so[] = { + "dup_libretro.so", "dup_libretro_android.so", + "same1_libretro.so", "same2_libretro.so", "slash_libretro.so", + "two_part_libretro.so", "twin_libretro.so", "Twin_libretro.so" + }; + char p[1024]; + size_t i; + FILE *f; + + for (i = 0; i < sizeof(info) / sizeof(info[0]); i++) + { + sprintf(p, "%s/%s_libretro.info", info_dir, info[i][0]); + if (!(f = fopen(p, "w"))) + return false; + fprintf(f, "display_name = \"%s\"\nsupported_extensions = \"%s\"\n", + info[i][1], info[i][2]); + fclose(f); + } + for (i = 0; i < sizeof(so) / sizeof(so[0]); i++) + { + sprintf(p, "%s/%s", core_dir, so[i]); + if (!(f = fopen(p, "wb"))) + return false; + fputs("\177ELF", f); + fclose(f); + } + return true; +} + +/* The core list's path for the core whose file is @file. */ +static const char *lookup_core_path(const char *file) +{ + core_info_list_t *list = NULL; + size_t i; + + core_info_get_list(&list); + for (i = 0; list && i < list->count; i++) + { + core_info_t *info = core_info_get(list, i); + if (info && info->path && !strcmp(path_basename(info->path), file)) + return info->path; + } + return ""; +} + +static char *ovr_str(const char *s) +{ + char *d = (char*)malloc(strlen(s) + 1); + if (d) + strcpy(d, s); + return d; +} + +/* A fresh record in place of @ov, which is freed. */ +static manual_content_scan_overrides_t *ovr_fresh( + manual_content_scan_overrides_t *ov) +{ + manual_content_scan_overrides_free(ov); + return manual_content_scan_overrides_new(); +} + +static manual_content_scan_task_config_t *ovr_cfg; +static bool ovr_cfg_ok; +static const char *ovr_dir; +static const char *ovr_pl; + +/* A --scan item's settings: a reset, the folder, @ov. The task config + * lands in ovr_cfg, and ovr_cfg_ok says whether it could be built. */ +static enum manual_content_scan_override_status ovr_run( + const manual_content_scan_overrides_t *ov, + const struct string_list *rdb, unsigned *count) +{ + enum manual_content_scan_override_status st; + + manual_content_scan_reset_settings(); + manual_content_scan_set_menu_content_dir(ovr_dir); + st = manual_content_scan_apply_overrides(ov, rdb, count); + ovr_cfg_ok = st == MANUAL_CONTENT_SCAN_OVR_OK + && manual_content_scan_get_task_config(ovr_cfg, ovr_pl); + return st; +} + +/* After a Custom setup, every setting is back to its default, and + * Custom over a reset builds the task Fully Automatic does. */ +static void reset_lane(const char *root, const char *pl_dir) +{ + manual_content_scan_task_config_t *cfg = + (manual_content_scan_task_config_t*)malloc(sizeof(*cfg)); + char dir[512]; + bool set_up; + + if (!cfg) + { + check(0, "reset lane", "out of memory"); + return; + } + sprintf(dir, "%s/scan_reset", root); + path_mkdir(dir); + + manual_content_scan_set_menu_scan_method( + MANUAL_CONTENT_SCAN_METHOD_CUSTOM); + manual_content_scan_set_menu_content_dir(dir); + manual_content_scan_set_menu_scan_use_db( + MANUAL_CONTENT_SCAN_USE_DB_DAT_LOOSE); + strlcpy(manual_content_scan_get_dat_file_path_ptr(), "/x/reset.dat", + manual_content_scan_get_dat_file_path_size()); + manual_content_scan_set_menu_scan_db_select( + MANUAL_CONTENT_SCAN_SELECT_DB_SPECIFIC, "Test Alpha"); + strlcpy(manual_content_scan_get_system_name_custom_ptr(), "Reset Custom", + manual_content_scan_get_system_name_custom_size()); + manual_content_scan_set_menu_system_name( + MANUAL_CONTENT_SCAN_SYSTEM_NAME_CUSTOM, NULL); + set_up = manual_content_scan_set_menu_core_name( + MANUAL_CONTENT_SCAN_CORE_SET, "Scan Test") + && !strcmp(manual_content_scan_get_file_exts(), "bin cue gdi m3u zip"); + strlcpy(manual_content_scan_get_file_exts_custom_ptr(), "rst", + manual_content_scan_get_file_exts_custom_size()); + *manual_content_scan_get_search_recursively_ptr() = false; + *manual_content_scan_get_search_archives_ptr() = false; + *manual_content_scan_get_omit_db_ref_ptr() = true; + *manual_content_scan_get_filter_dat_content_ptr() = true; + *manual_content_scan_get_overwrite_playlist_ptr() = true; + *manual_content_scan_get_validate_entries_ptr() = true; + check(set_up, "reset: the Custom setup took", "core Scan Test"); + + manual_content_scan_reset_settings(); + check( manual_content_scan_get_scan_method_enum() + == MANUAL_CONTENT_SCAN_METHOD_AUTOMATIC + && manual_content_scan_get_scan_use_db_enum() + == MANUAL_CONTENT_SCAN_USE_DB_STRICT + && manual_content_scan_get_scan_db_select_enum() + == MANUAL_CONTENT_SCAN_SELECT_DB_AUTO + && manual_content_scan_get_menu_system_name_type() + == MANUAL_CONTENT_SCAN_SYSTEM_NAME_AUTO, + "reset: method, database, playlist", "Automatic defaults"); + check( !*manual_content_scan_get_content_dir_ptr() + && !*manual_content_scan_get_dat_file_path_ptr() + && !*manual_content_scan_get_system_name_custom_ptr() + && !*manual_content_scan_get_file_exts_custom_ptr() + && !*manual_content_scan_get_file_exts(), + "reset: every path and name emptied", ""); + check( *manual_content_scan_get_search_recursively_ptr() + && *manual_content_scan_get_search_archives_ptr() + && !*manual_content_scan_get_omit_db_ref_ptr() + && !*manual_content_scan_get_filter_dat_content_ptr() + && !*manual_content_scan_get_overwrite_playlist_ptr() + && !*manual_content_scan_get_validate_entries_ptr(), + "reset: the switches", "recursive, archives on, rest off"); + + manual_content_scan_set_menu_content_dir(dir); + manual_content_scan_set_menu_scan_method( + MANUAL_CONTENT_SCAN_METHOD_CUSTOM); + check( manual_content_scan_get_task_config(cfg, pl_dir) + && !cfg->core_set && !*cfg->core_name && !*cfg->core_path + && !*cfg->file_exts && !cfg->file_exts_custom_set + && !*cfg->system_name && !*cfg->playlist_file + && !*cfg->database_name + && cfg->search_recursively && cfg->search_archives + && !cfg->filter_dat_content && !cfg->overwrite_playlist + && !cfg->validate_entries && !cfg->omit_db_reference + && cfg->db_usage == MANUAL_CONTENT_SCAN_USE_DB_STRICT + && cfg->db_selection == MANUAL_CONTENT_SCAN_SELECT_DB_AUTO, + "reset: Custom builds the automatic task", ""); + + manual_content_scan_reset_settings(); + free(cfg); +} + +/* Each override lands in the task config. */ +static void overrides_lane(const char *root, const char *pl_dir) +{ + manual_content_scan_overrides_t *ov = NULL; + struct string_list *rdb = string_list_new(); + union string_list_elem_attr attr; + char dir[512], dat[1024], txt[1024], empty[1024], missing[1024]; + char p[1024]; + unsigned count = 0; + enum manual_content_scan_override_status st; + FILE *f; + + ovr_cfg = (manual_content_scan_task_config_t*)malloc(sizeof(*ovr_cfg)); + if (!rdb || !ovr_cfg) + { + check(0, "overrides lane", "out of memory"); + goto end; + } + attr.i = 0; + string_list_append(rdb, "/db/Test Alpha.rdb", attr); + string_list_append(rdb, "/db/Test Beta.rdb", attr); + + sprintf(dir, "%s/scan_ovr", root); + sprintf(dat, "%s/scan_ovr/ovr.dat", root); + sprintf(txt, "%s/scan_ovr/ovr.txt", root); + sprintf(empty, "%s/scan_ovr/empty.dat", root); + sprintf(missing, "%s/scan_ovr/missing.dat", root); + path_mkdir(dir); + if (!(f = fopen(dat, "w"))) + goto fixture; + fputs("\n\n", f); + fclose(f); + if (!(f = fopen(txt, "w"))) + goto fixture; + fputs("not a dat\n", f); + fclose(f); + if (!(f = fopen(empty, "w"))) + goto fixture; + fclose(f); + remove(missing); + ovr_dir = dir; + ovr_pl = pl_dir; + + check( ovr_run(NULL, rdb, &count) == MANUAL_CONTENT_SCAN_OVR_OK + && manual_content_scan_get_scan_method_enum() + == MANUAL_CONTENT_SCAN_METHOD_AUTOMATIC, + "overrides: none leave Fully Automatic", ""); + + /* Core: path, name and extensions */ + ov = ovr_fresh(ov); + ov->core = ovr_str("Scan Test"); + strlcpy(p, lookup_core_path("test_libretro.so"), sizeof(p)); + st = ovr_run(ov, rdb, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok + && manual_content_scan_get_scan_method_enum() + == MANUAL_CONTENT_SCAN_METHOD_CUSTOM + && ovr_cfg->core_set && count == 1 + && !strcmp(ovr_cfg->core_name, "Scan Test") + && !strcmp(ovr_cfg->core_path, p) + && !strcmp(ovr_cfg->file_exts, "bin|cue|gdi|m3u|zip") + && !ovr_cfg->file_exts_custom_set, + "overrides: core, its path and extensions", ovr_cfg->core_path); + + /* Custom extensions win over the core's, scrubbed */ + ov->exts = ovr_str(".BIN gam"); + st = ovr_run(ov, rdb, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok + && ovr_cfg->file_exts_custom_set + && !strcmp(ovr_cfg->file_exts, "bin|gam"), + "overrides: extensions", ovr_cfg->file_exts); + + /* Database Check without a DAT */ + ov = ovr_fresh(ov); + ov->flags = MANUAL_CONTENT_SCAN_OVR_MATCH; + ov->match = MANUAL_CONTENT_SCAN_USE_DB_LOOSE; + st = ovr_run(ov, rdb, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok + && ovr_cfg->db_usage == MANUAL_CONTENT_SCAN_USE_DB_LOOSE, + "overrides: match loose", ""); + ov->match = MANUAL_CONTENT_SCAN_USE_DB_STRICT; + st = ovr_run(ov, rdb, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok + && ovr_cfg->db_usage == MANUAL_CONTENT_SCAN_USE_DB_STRICT, + "overrides: match strict", ""); + ov->match = MANUAL_CONTENT_SCAN_USE_DB_NONE; + sprintf(p, "%s/scan_ovr.lpl", pl_dir); + st = ovr_run(ov, rdb, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok + && ovr_cfg->db_usage == MANUAL_CONTENT_SCAN_USE_DB_NONE + && !strcmp(ovr_cfg->system_name, "scan_ovr") + && !strcmp(ovr_cfg->playlist_file, p), + "overrides: match none, named after the folder", + ovr_cfg->system_name); + + /* A DAT: strict unless loose, never with none */ + ov = ovr_fresh(ov); + ov->dat = ovr_str(dat); + st = ovr_run(ov, rdb, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok + && ovr_cfg->db_usage == MANUAL_CONTENT_SCAN_USE_DB_DAT_STRICT + && ovr_cfg->filter_dat_content + && !strcmp(ovr_cfg->dat_file_path, dat), + "overrides: DAT alone is strict", ovr_cfg->dat_file_path); + ov->flags = MANUAL_CONTENT_SCAN_OVR_MATCH; + ov->match = MANUAL_CONTENT_SCAN_USE_DB_LOOSE; + st = ovr_run(ov, rdb, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok + && ovr_cfg->db_usage == MANUAL_CONTENT_SCAN_USE_DB_DAT_LOOSE + && !ovr_cfg->filter_dat_content + && !strcmp(ovr_cfg->dat_file_path, dat), + "overrides: DAT with loose", ""); + ov->match = MANUAL_CONTENT_SCAN_USE_DB_STRICT; + st = ovr_run(ov, rdb, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok + && ovr_cfg->db_usage == MANUAL_CONTENT_SCAN_USE_DB_DAT_STRICT, + "overrides: DAT with strict", ""); + ov->match = MANUAL_CONTENT_SCAN_USE_DB_NONE; + st = ovr_run(ov, rdb, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_DAT_WITHOUT_MATCH + && !*manual_content_scan_get_dat_file_path_ptr(), + "overrides: DAT with none fails untouched", ""); + + /* DAT errors */ + ov = ovr_fresh(ov); + ov->dat = ovr_str(txt); + check(ovr_run(ov, rdb, &count) == MANUAL_CONTENT_SCAN_OVR_NOT_A_DAT, + "overrides: DAT with another extension", "not a DAT"); + ov = ovr_fresh(ov); + ov->dat = ovr_str(missing); + check(ovr_run(ov, rdb, &count) == MANUAL_CONTENT_SCAN_OVR_NOT_A_DAT, + "overrides: DAT missing", "not a DAT"); + ov = ovr_fresh(ov); + ov->dat = ovr_str(empty); + check(ovr_run(ov, rdb, &count) == MANUAL_CONTENT_SCAN_OVR_NOT_A_DAT, + "overrides: DAT empty", "not a DAT"); + + /* A refresh's saved DAT: the match picks its mode, none drops it */ + ov = ovr_fresh(ov); + ov->flags = MANUAL_CONTENT_SCAN_OVR_MATCH; + { + static const enum manual_content_scan_db_usage + match[3] = { MANUAL_CONTENT_SCAN_USE_DB_LOOSE, + MANUAL_CONTENT_SCAN_USE_DB_STRICT, + MANUAL_CONTENT_SCAN_USE_DB_NONE }; + static const enum manual_content_scan_db_usage + want[3] = { MANUAL_CONTENT_SCAN_USE_DB_DAT_LOOSE, + MANUAL_CONTENT_SCAN_USE_DB_DAT_STRICT, + MANUAL_CONTENT_SCAN_USE_DB_NONE }; + static const char *const what[3] = { + "overrides: saved DAT with loose", + "overrides: saved DAT with strict", + "overrides: saved DAT with none drops it" }; + int i; + + for (i = 0; i < 3; i++) + { + manual_content_scan_reset_settings(); + manual_content_scan_set_menu_content_dir(dir); + manual_content_scan_set_menu_scan_use_db( + MANUAL_CONTENT_SCAN_USE_DB_DAT_STRICT); + strlcpy(manual_content_scan_get_dat_file_path_ptr(), dat, + manual_content_scan_get_dat_file_path_size()); + ov->match = match[i]; + st = manual_content_scan_apply_overrides(ov, rdb, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK + && manual_content_scan_get_task_config(ovr_cfg, pl_dir) + && ovr_cfg->db_usage == want[i] + && (i == 2 + ? !*manual_content_scan_get_dat_file_path_ptr() + : !strcmp(ovr_cfg->dat_file_path, dat)), + what[i], ""); + } + } + + /* Database To Match */ + ov = ovr_fresh(ov); + ov->database = ovr_str("Test Alpha"); + sprintf(p, "%s/Test Alpha.lpl", pl_dir); + st = ovr_run(ov, rdb, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok + && ovr_cfg->db_selection == MANUAL_CONTENT_SCAN_SELECT_DB_SPECIFIC + && !strcmp(ovr_cfg->database_name, "Test Alpha.lpl") + && !strcmp(ovr_cfg->system_name, "Test Alpha") + && !strcmp(ovr_cfg->playlist_file, p), + "overrides: database by name", ovr_cfg->database_name); + check(ovr_run(ov, NULL, &count) == MANUAL_CONTENT_SCAN_OVR_NO_SUCH_DATABASE, + "overrides: no database installed", "no such database"); + ov = ovr_fresh(ov); + ov->database = ovr_str("Test Nope"); + check(ovr_run(ov, rdb, &count) == MANUAL_CONTENT_SCAN_OVR_NO_SUCH_DATABASE, + "overrides: an unknown database", "no such database"); + ov = ovr_fresh(ov); + ov->flags = MANUAL_CONTENT_SCAN_OVR_FIRST_DB; + st = ovr_run(ov, rdb, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok + && ovr_cfg->db_selection + == MANUAL_CONTENT_SCAN_SELECT_DB_AUTO_FIRST_MATCH, + "overrides: first matching database", ""); + + /* Target Playlist */ + ov = ovr_fresh(ov); + ov->playlist = ovr_str("Test Beta"); + sprintf(p, "%s/Test Beta.lpl", pl_dir); + st = ovr_run(ov, rdb, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok + && manual_content_scan_get_menu_system_name_type() + == MANUAL_CONTENT_SCAN_SYSTEM_NAME_DATABASE + && !strcmp(ovr_cfg->system_name, "Test Beta") + && !strcmp(ovr_cfg->playlist_file, p), + "overrides: playlist named after a database", ovr_cfg->system_name); + ov->database = ovr_str("Test Alpha"); + st = ovr_run(ov, rdb, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok + && manual_content_scan_get_menu_system_name_type() + == MANUAL_CONTENT_SCAN_SYSTEM_NAME_CUSTOM + && !strcmp(ovr_cfg->system_name, "Test Beta") + && !strcmp(ovr_cfg->playlist_file, p) + && !strcmp(ovr_cfg->database_name, "Test Alpha.lpl"), + "overrides: and another database: Custom", ovr_cfg->database_name); + free(ov->playlist); + ov->playlist = ovr_str("Test Alpha"); + st = ovr_run(ov, rdb, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok + && manual_content_scan_get_menu_system_name_type() + == MANUAL_CONTENT_SCAN_SYSTEM_NAME_DATABASE + && !strcmp(ovr_cfg->system_name, "Test Alpha"), + "overrides: and the same database", ovr_cfg->system_name); + /* As a refresh of a database playlist leaves them */ + ov = ovr_fresh(ov); + ov->database = ovr_str("Test Alpha"); + sprintf(p, "%s/Test Beta.lpl", pl_dir); + manual_content_scan_reset_settings(); + manual_content_scan_set_menu_content_dir(ovr_dir); + manual_content_scan_set_menu_system_name( + MANUAL_CONTENT_SCAN_SYSTEM_NAME_DATABASE, "Test Beta"); + manual_content_scan_set_menu_scan_db_select( + MANUAL_CONTENT_SCAN_SELECT_DB_SPECIFIC, "Test Beta"); + st = manual_content_scan_apply_overrides(ov, rdb, &count); + ovr_cfg_ok = st == MANUAL_CONTENT_SCAN_OVR_OK + && manual_content_scan_get_task_config(ovr_cfg, pl_dir); + check( ovr_cfg_ok + && !strcmp(ovr_cfg->playlist_file, p) + && !strcmp(ovr_cfg->database_name, "Test Alpha.lpl"), + "overrides: a refresh's database playlist keeps its name", + ovr_cfg_ok ? ovr_cfg->playlist_file : "no task config"); + ov = ovr_fresh(ov); + ov->playlist = ovr_str("My: List?"); + sprintf(p, "%s/My_ List_.lpl", pl_dir); + st = ovr_run(ov, rdb, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok + && manual_content_scan_get_menu_system_name_type() + == MANUAL_CONTENT_SCAN_SYSTEM_NAME_CUSTOM + && !strcmp(ovr_cfg->system_name, "My_ List_") + && !strcmp(ovr_cfg->playlist_file, p), + "overrides: custom playlist, scrubbed", ovr_cfg->system_name); + + /* The switches */ + ov = ovr_fresh(ov); + ov->flags = MANUAL_CONTENT_SCAN_OVR_NO_RECURSIVE + | MANUAL_CONTENT_SCAN_OVR_NO_ARCHIVES + | MANUAL_CONTENT_SCAN_OVR_OVERWRITE + | MANUAL_CONTENT_SCAN_OVR_VALIDATE + | MANUAL_CONTENT_SCAN_OVR_OMIT_DB_REF; + st = ovr_run(ov, rdb, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok + && !ovr_cfg->search_recursively && !ovr_cfg->search_archives + && ovr_cfg->overwrite_playlist && ovr_cfg->validate_entries + && ovr_cfg->omit_db_reference, + "overrides: recursive, archives, overwrite, validate, omit", ""); + + /* A step that fails stops the ones after it */ + ov = ovr_fresh(ov); + ov->core = ovr_str("No Such Core"); + ov->database = ovr_str("Test Alpha"); + st = ovr_run(ov, rdb, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_NO_SUCH_CORE && count == 0 + && manual_content_scan_get_scan_db_select_enum() + == MANUAL_CONTENT_SCAN_SELECT_DB_AUTO + && manual_content_scan_get_scan_method_enum() + == MANUAL_CONTENT_SCAN_METHOD_AUTOMATIC, + "overrides: an error stops the rest", ""); + goto end; + +fixture: + check(0, "fixture", "could not write the DAT files"); +end: + manual_content_scan_overrides_free(ov); + manual_content_scan_reset_settings(); + string_list_free(rdb); + free(ovr_cfg); + ovr_cfg = NULL; +} + +/* --scan-core: a path, then a file name, then a display name; more + * than one match in the step that decides is ambiguous. */ +static void core_lookup_lane(const char *root) +{ + manual_content_scan_overrides_t *ov = NULL; + char elsewhere[512], p[1024]; + char dup_path[1024]; + unsigned count; + size_t i; + enum manual_content_scan_override_status st; + const char *core_name = NULL; + static const char *const by_name[] = { + "gamma", "gamma_libretro", "gamma_libretro.so", "gamma_libretro.dll", + "gamma_libretro_android.so", "Gamma Test" }; + static const char *const two_part[] = { + "two_part", "two_part_libretro_ios.dylib" }; + static const char *const other_case[] = { + "GAMMA", "Gamma_libretro.so", "Gamma_Libretro.dll", + "GAMMA_LIBRETRO_ANDROID.SO" }; + /* What each name finds; "" for both */ + static const char *const twin[][2] = { + { "twin", "twin_libretro.so" }, + { "Twin_libretro.so", "Twin_libretro.so" }, + { "TWIN", "" } }; + bool twins; + FILE *f; + + ovr_cfg = (manual_content_scan_task_config_t*)malloc(sizeof(*ovr_cfg)); + if (!ovr_cfg) + { + check(0, "core lookup lane", "out of memory"); + return; + } + sprintf(elsewhere, "%s/scan_lookup_elsewhere", root); + path_mkdir(elsewhere); + sprintf(p, "%s/gamma_libretro.so", elsewhere); + if ((f = fopen(p, "wb"))) + { + fputs("\177ELF", f); + fclose(f); + } + ovr_dir = elsewhere; + ovr_pl = elsewhere; + + ov = ovr_fresh(ov); + ov->core = ovr_str(lookup_core_path("gamma_libretro.so")); + count = 9; + st = ovr_run(ov, NULL, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok && count == 1 + && !strcmp(ovr_cfg->core_name, "Gamma Test") + && !strcmp(ovr_cfg->core_path, ov->core), + "core lookup: by path", ovr_cfg->core_path); + + free(ov->core); + ov->core = ovr_str(p); + count = 9; + check( ovr_run(ov, NULL, &count) == MANUAL_CONTENT_SCAN_OVR_NO_SUCH_CORE + && count == 0, + "core lookup: a path outside the core folder", "no such core"); + + for (i = 0; i < sizeof(by_name) / sizeof(by_name[0]); i++) + { + free(ov->core); + ov->core = ovr_str(by_name[i]); + st = ovr_run(ov, NULL, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok && count == 1 + && !strcmp(ovr_cfg->core_name, "Gamma Test"), + "core lookup: by file or display name", by_name[i]); + } + + for (i = 0; i < sizeof(two_part) / sizeof(two_part[0]); i++) + { + free(ov->core); + ov->core = ovr_str(two_part[i]); + st = ovr_run(ov, NULL, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok && count == 1 + && !strcmp(ovr_cfg->core_name, "Two Part Test"), + "core lookup: a file id with an underscore", two_part[i]); + } + + /* File names ignore case, display names do not */ + for (i = 0; i < sizeof(other_case) / sizeof(other_case[0]); i++) + { + free(ov->core); + ov->core = ovr_str(other_case[i]); + count = 9; + st = ovr_run(ov, NULL, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok && count == 1 + && !strcmp(ovr_cfg->core_name, "Gamma Test"), + "core lookup: a file name in another case", other_case[i]); + } + free(ov->core); + ov->core = ovr_str("gamma test"); + count = 9; + check( ovr_run(ov, NULL, &count) == MANUAL_CONTENT_SCAN_OVR_NO_SUCH_CORE + && count == 0, + "core lookup: a display name in another case", "no such core"); + + free(ov->core); + ov->core = ovr_str("Slash / Test (Lookup)"); + st = ovr_run(ov, NULL, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok + && !strcmp(ovr_cfg->core_name, "Slash / Test (Lookup)"), + "core lookup: a display name with a slash", ovr_cfg->core_name); + + free(ov->core); + ov->core = ovr_str("nope"); + count = 9; + check( ovr_run(ov, NULL, &count) == MANUAL_CONTENT_SCAN_OVR_NO_SUCH_CORE + && count == 0 + && manual_content_scan_get_menu_core_name(&core_name) + && manual_content_scan_get_scan_method_enum() + == MANUAL_CONTENT_SCAN_METHOD_AUTOMATIC, + "core lookup: unknown", "no such core"); + + /* dup_libretro.so and dup_libretro_android.so share a file id */ + free(ov->core); + ov->core = ovr_str("dup"); + check( ovr_run(ov, NULL, &count) == MANUAL_CONTENT_SCAN_OVR_AMBIGUOUS_CORE + && count == 2, + "core lookup: two files with one file id", "2 cores"); + free(ov->core); + ov->core = ovr_str("dup_libretro.so"); + check( ovr_run(ov, NULL, &count) == MANUAL_CONTENT_SCAN_OVR_AMBIGUOUS_CORE + && count == 2, + "core lookup: and by its file name", "2 cores"); + free(ov->core); + ov->core = ovr_str("dup_libretro_android.so"); + check( ovr_run(ov, NULL, &count) == MANUAL_CONTENT_SCAN_OVR_AMBIGUOUS_CORE + && count == 2, + "core lookup: and by its platform file name", "2 cores"); + strlcpy(dup_path, lookup_core_path("dup_libretro_android.so"), + sizeof(dup_path)); + free(ov->core); + ov->core = ovr_str(dup_path); + st = ovr_run(ov, NULL, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok && count == 1 + && !strcmp(ovr_cfg->core_path, dup_path), + "core lookup: the path tells them apart", path_basename(dup_path)); +#ifdef _WIN32 + /* All '\', '\' before the file name only, and all '\' in capitals */ + for (i = 0; i < 3; i++) + { + char *slash; + + strlcpy(p, dup_path, sizeof(p)); + if (i != 1) + pathname_conform_slashes_to_os(p); + else if ((slash = strrchr(p, '/'))) + *slash = '\\'; + if (i == 2) + string_to_upper(p); + free(ov->core); + ov->core = ovr_str(p); + count = 9; + st = ovr_run(ov, NULL, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok && count == 1 + && !strcmp(ovr_cfg->core_path, dup_path), + "core lookup: the path with either slash", p); + } +#endif + + /* The exact case first, else both are ambiguous; a file system + * that ignores case keeps one of the two */ + twins = *lookup_core_path("twin_libretro.so") + && *lookup_core_path("Twin_libretro.so"); + for (i = 0; i < sizeof(twin) / sizeof(twin[0]); i++) + { + free(ov->core); + ov->core = ovr_str(twin[i][0]); + count = 9; + st = ovr_run(ov, NULL, &count); + if (!twins) + check(st == MANUAL_CONTENT_SCAN_OVR_OK && count == 1, + "core lookup: two file names in another case", "one file"); + else if (*twin[i][1]) + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok && count == 1 + && !strcmp(ovr_cfg->core_path, lookup_core_path(twin[i][1])), + "core lookup: the file name in the case typed", twin[i][0]); + else + check(st == MANUAL_CONTENT_SCAN_OVR_AMBIGUOUS_CORE && count == 2, + "core lookup: two file names in another case", twin[i][0]); + } + + free(ov->core); + ov->core = ovr_str("Same Name Test"); + check( ovr_run(ov, NULL, &count) == MANUAL_CONTENT_SCAN_OVR_AMBIGUOUS_CORE + && count == 2, + "core lookup: two cores with one display name", "2 cores"); + free(ov->core); + ov->core = ovr_str("same2"); + st = ovr_run(ov, NULL, &count); + check( st == MANUAL_CONTENT_SCAN_OVR_OK && ovr_cfg_ok && count == 1 + && !strcmp(path_basename(ovr_cfg->core_path), "same2_libretro.so"), + "core lookup: the file name tells them apart", + path_basename(ovr_cfg->core_path)); + + manual_content_scan_overrides_free(ov); + manual_content_scan_reset_settings(); + free(ovr_cfg); + ovr_cfg = NULL; +} + int main(int argc, char **argv) { const char *root = (argc > 1) ? argv[1] : "/tmp"; @@ -8669,6 +9345,8 @@ int main(int argc, char **argv) } sprintf(p, "%s/rarcart_libretro.so", core_dir); { FILE *f = fopen(p, "wb"); if (f) { fputs("\177ELF", f); fclose(f); } } + if (!write_lookup_cores(info_dir, core_dir)) + { check(0, "fixture", "could not write core info"); return 1; } #ifdef HAVE_THREADS task_queue_init(true, msgq_push); @@ -9476,6 +10154,9 @@ int main(int argc, char **argv) failed_lane(root, pl_dir, db_dir, crc_a, sz_a); errors_lane(root, pl_dir, crc_a, sz_a); output_lane(root, pl_dir, db_dir); + reset_lane(root, pl_dir); + overrides_lane(root, pl_dir); + core_lookup_lane(root); { char reader_db[512]; From a8402d9345ef6c9cf0b62afa4f57fe702117a10b Mon Sep 17 00:00:00 2001 From: davey Date: Sat, 31 Oct 2026 19:12:40 -0700 Subject: [PATCH 12/21] scan: run --scan items through one chain The --scan paths ran through a loop in retroarch.c, and with --menu through a second chain of their own. The scan options and playlist refreshes to come need one runner for every item, from the command line and from the menu. task_push_dbscan_chain() runs a list of items one task at a time: a prep task lists the databases, then each step is pushed from the last task's callback on the main thread, so nothing waits and nothing locks. An item that fails is reported and the run goes on; a cancelled task ends it. retroarch.c builds the items in the first argument pass and hands them to the chain; without --menu it waits for the chain's end and exits as before. Running out of memory prints "out of memory" on stderr: for an item's path the item fails, and for the chain itself nothing runs, which without --menu exits 1. The chain also prints a scan's override errors, and the warnings for an option that needs one playlist or does nothing for a single file, which the scan options of the next commit can trigger. --- manual_content_scan.c | 36 +- manual_content_scan.h | 4 + retroarch.c | 194 ++++------ samples/tasks/database/database_scan_test.c | 289 ++++++++++++++- tasks/task_database.c | 381 ++++++++++++++++++++ tasks/tasks_internal.h | 49 +++ 6 files changed, 820 insertions(+), 133 deletions(-) diff --git a/manual_content_scan.c b/manual_content_scan.c index 8b6d4e4145be..b9d7672d55c1 100644 --- a/manual_content_scan.c +++ b/manual_content_scan.c @@ -1703,6 +1703,32 @@ struct string_list *manual_content_scan_get_menu_core_name_list(void) /* Task Helpers */ /****************/ +/* The system name type a scan of the current settings takes: an + * automatic name is filled when the settings predict it. */ +static enum manual_content_scan_system_name_type + manual_content_scan_task_name_type(void) +{ + if (scan_settings.system_name_type == MANUAL_CONTENT_SCAN_SYSTEM_NAME_AUTO) + { + if (scan_settings.db_usage == MANUAL_CONTENT_SCAN_USE_DB_DAT_STRICT || + scan_settings.db_usage == MANUAL_CONTENT_SCAN_USE_DB_DAT_LOOSE || + scan_settings.db_usage == MANUAL_CONTENT_SCAN_USE_DB_NONE) + return MANUAL_CONTENT_SCAN_SYSTEM_NAME_CONTENT_DIR; + if (scan_settings.db_selection == MANUAL_CONTENT_SCAN_SELECT_DB_SPECIFIC) + return MANUAL_CONTENT_SCAN_SYSTEM_NAME_DATABASE; + } + return scan_settings.system_name_type; +} + +/* True if a scan of the current settings writes one playlist + * (target_is_single_determined_playlist). */ +bool manual_content_scan_is_single_playlist(void) +{ + return scan_settings.scan_method != MANUAL_CONTENT_SCAN_METHOD_AUTOMATIC + && manual_content_scan_task_name_type() + != MANUAL_CONTENT_SCAN_SYSTEM_NAME_AUTO; +} + /* Parses current manual content scan settings, * and extracts all information required to configure * a manual content scan task. @@ -1749,15 +1775,7 @@ bool manual_content_scan_get_task_config( sizeof(task_config->content_dir)); /* If name is set as automatic, but is predictable, fill it now. */ - if (scan_settings.system_name_type == MANUAL_CONTENT_SCAN_SYSTEM_NAME_AUTO) - { - if (scan_settings.db_usage == MANUAL_CONTENT_SCAN_USE_DB_DAT_STRICT || - scan_settings.db_usage == MANUAL_CONTENT_SCAN_USE_DB_DAT_LOOSE || - scan_settings.db_usage == MANUAL_CONTENT_SCAN_USE_DB_NONE) - scan_settings.system_name_type = MANUAL_CONTENT_SCAN_SYSTEM_NAME_CONTENT_DIR; - else if (scan_settings.db_selection == MANUAL_CONTENT_SCAN_SELECT_DB_SPECIFIC) - scan_settings.system_name_type = MANUAL_CONTENT_SCAN_SYSTEM_NAME_DATABASE; - } + scan_settings.system_name_type = manual_content_scan_task_name_type(); /* Get target playlist ("system name") * * The system name is not a leaf value: the scan pipeline appends a diff --git a/manual_content_scan.h b/manual_content_scan.h index 1738e03f1172..776a9af73134 100644 --- a/manual_content_scan.h +++ b/manual_content_scan.h @@ -371,6 +371,10 @@ struct string_list *manual_content_scan_get_menu_core_name_list(void); /* Task Helpers */ /****************/ +/* True if a scan of the current settings writes one playlist + * (target_is_single_determined_playlist). */ +bool manual_content_scan_is_single_playlist(void); + /* Parses current manual content scan settings, * and extracts all information required to configure * a manual content scan task. diff --git a/retroarch.c b/retroarch.c index 4cc58d0918b9..b4953daafae4 100644 --- a/retroarch.c +++ b/retroarch.c @@ -8261,111 +8261,73 @@ void handle_dbscan_finished(retro_task_t *task, #endif #ifdef HAVE_LIBRETRODB -/* @in with '~' expanded, made absolute if relative; NULL, with the - * reason on stderr, if nothing is there. Free the result. */ -static char *retroarch_cli_scan_path(const char *in) +/* Appends a @kind item for @arg; false if out of memory. */ +static bool retroarch_cli_scan_add(task_dbscan_item_t **items, + size_t *count, size_t *cap, enum task_dbscan_item_kind kind, + const char *arg) { - char *s = (char*)malloc(PATH_MAX_LENGTH); + task_dbscan_item_t *item; - if (!s) - return NULL; - fill_pathname_expand_special(s, in, PATH_MAX_LENGTH); - /* "" would resolve to the working directory. Consoles resolve - * nothing, so a relative path stays as it is there. */ -#ifndef RARCH_CONSOLE - if ( *s && !path_is_absolute(s) - && !path_resolve_realpath(s, PATH_MAX_LENGTH, false)) - *s = '\0'; -#endif - if (!*s || !path_is_valid(s)) - { - fprintf(stderr, "%s: not found\n", in); - free(s); - return NULL; - } - return s; + if (*count == *cap) + { + size_t n = *cap ? *cap * 2 : 4; + task_dbscan_item_t *grown = (task_dbscan_item_t*)realloc(*items, + n * sizeof(*grown)); + + if (!grown) + return false; + *items = grown; + *cap = n; + } + item = &(*items)[*count]; + item->kind = kind; + item->overrides = NULL; + item->arg = NULL; + if (arg && !(item->arg = strdup(arg))) + return false; + (*count)++; + return true; } -static void retroarch_cli_scan_done(retro_task_t *task, - void *task_data, void *user_data, const char *err) +/* What the chain told a run without --menu. */ +typedef struct { - *(task_dbscan_result_t*)user_data = - *(const task_dbscan_result_t*)task_data; -} + task_dbscan_chain_result_t totals; + bool done; +} retroarch_cli_scan_state_t; -/* Scans @in to its end on this thread. */ -static void retroarch_cli_scan_one(const char *in, - task_dbscan_result_t *res) +static void retroarch_cli_scan_done( + const task_dbscan_chain_result_t *res, void *user_data) { - char *path = retroarch_cli_scan_path(in); + retroarch_cli_scan_state_t *st = (retroarch_cli_scan_state_t*)user_data; - memset(res, 0, sizeof(*res)); - res->failed = true; - if (!path) - return; - if (task_push_dbscan(path, retroarch_cli_scan_done, res)) - task_queue_wait(NULL, NULL); - else - fprintf(stderr, "%s: the scan could not start\n", path); - free(path); + st->totals = *res; + st->done = true; } -/* --scan with --menu: one scan at a time, as a second is refused - * while one runs; each pushes the next as it finishes. */ -typedef struct +static bool retroarch_cli_scan_running(void *data) { - struct string_list *paths; - size_t next; -} retroarch_menu_scans_t; - -static void retroarch_menu_scan_next(retroarch_menu_scans_t *scans); + return !((retroarch_cli_scan_state_t*)data)->done; +} -static void retroarch_menu_scan_done(retro_task_t *task, - void *task_data, void *user_data, const char *err) +static void retroarch_cli_scan_menu_item(const char *name, + enum task_dbscan_outcome outcome, + const task_dbscan_result_t *res, void *user_data) { - retroarch_menu_scans_t *scans = (retroarch_menu_scans_t*)user_data; - #ifdef HAVE_MENU - handle_dbscan_finished(task, task_data, NULL, err); + if (res) + handle_dbscan_finished(NULL, NULL, NULL, NULL); #endif - /* Cancelled, as on quit: start no more. */ - if (task_get_flags(task) & RETRO_TASK_FLG_CANCELLED) - scans->next = scans->paths->size; - retroarch_menu_scan_next(scans); } -/* Pushes the next path that starts; frees @scans after the last. */ -static void retroarch_menu_scan_next(retroarch_menu_scans_t *scans) +/* Runs the --scan @items through the scan chain, which owns them. + * With --menu the chain is pushed and the menu goes on. Without, it + * is the whole run: the process exits 1 if any scan failed, else 0 if + * any added an entry, else 2. */ +static void retroarch_cli_scan(task_dbscan_item_t *items, size_t count, + bool menu, const char *content) { - while (scans->next < scans->paths->size) - { - bool pushed; - char *path = retroarch_cli_scan_path( - scans->paths->elems[scans->next++].data); - - if (!path) - continue; - pushed = task_push_dbscan(path, retroarch_menu_scan_done, scans); - if (!pushed) - fprintf(stderr, "%s: the scan could not start\n", path); - free(path); - if (pushed) - return; - } - string_list_free(scans->paths); - free(scans); -} - -/* Runs the --scan @paths, which it frees. With --menu the first is - * pushed and the rest follow it. Without, they are the whole run: - * each is scanned in turn, then the process exits 1 if any failed, - * else 0 if any added an entry, else 2. */ -static void retroarch_cli_scan(struct string_list *paths, bool menu, - const char *content) -{ - size_t i; - bool failed = false; - bool added = false; + retroarch_cli_scan_state_t st; settings_t *settings = config_get_ptr(); int reinit_flags = DRIVERS_CMD_ALL & ~(DRIVER_VIDEO_MASK | DRIVER_AUDIO_MASK | DRIVER_MICROPHONE_MASK | DRIVER_INPUT_MASK | DRIVER_MIDI_MASK); @@ -8375,16 +8337,9 @@ static void retroarch_cli_scan(struct string_list *paths, bool menu, if (menu) { - retroarch_menu_scans_t *scans = (retroarch_menu_scans_t*) - calloc(1, sizeof(*scans)); - - if (!scans) - { - string_list_free(paths); - return; - } - scans->paths = paths; - retroarch_menu_scan_next(scans); + if (!task_push_dbscan_chain(items, count, + retroarch_cli_scan_menu_item, NULL, NULL)) + fprintf(stderr, "out of memory\n"); return; } @@ -8402,21 +8357,16 @@ static void retroarch_cli_scan(struct string_list *paths, bool menu, fprintf(stderr, "without --menu: ignoring \"%s\"\n", content); - for (i = 0; i < paths->size; i++) - { - task_dbscan_result_t res; - - retroarch_cli_scan_one(paths->elems[i].data, &res); - if (res.failed) - failed = true; - else if (res.added) - added = true; - } - string_list_free(paths); + memset(&st, 0, sizeof(st)); + if (task_push_dbscan_chain(items, count, NULL, + retroarch_cli_scan_done, &st)) + task_queue_wait(retroarch_cli_scan_running, &st); + else + fprintf(stderr, "out of memory\n"); driver_uninit(DRIVERS_CMD_ALL, (enum driver_lifetime_flags)0); task_queue_deinit(); - exit(failed ? 1 : added ? 0 : 2); + exit((!st.done || st.totals.failed) ? 1 : st.totals.added ? 0 : 2); } #endif @@ -8473,7 +8423,9 @@ static bool retroarch_parse_input_and_config( access_state_t *access_st = access_state_get_ptr(); #endif #ifdef HAVE_LIBRETRODB - struct string_list *scan_paths = NULL; + task_dbscan_item_t *scan_items = NULL; + size_t scan_count = 0; + size_t scan_cap = 0; const char *scan_conflict = NULL; bool cli_scan = false; #endif @@ -8736,6 +8688,12 @@ static bool retroarch_parse_input_and_config( cli_scan = true; verbosity_enable(); retroarch_override_setting_set(RARCH_OVERRIDE_SETTING_DATABASE_SCAN, NULL); + if (!retroarch_cli_scan_add(&scan_items, &scan_count, + &scan_cap, TASK_DBSCAN_ITEM_PATH, optarg)) + { + fprintf(stderr, "out of memory\n"); + exit(1); + } #endif break; #if defined(HAVE_LIBRETRODB) && defined(HAVE_NETWORKING) && defined(HAVE_NETWORK_CMD) && defined(HAVE_COMMAND) @@ -9203,17 +9161,7 @@ static bool retroarch_parse_input_and_config( } break; case RA_OPT_DATABASE_SCAN: -#ifdef HAVE_LIBRETRODB - { - union string_list_elem_attr attr; - - attr.i = 0; - if (!scan_paths) - scan_paths = string_list_new(); - if (scan_paths) - string_list_append(scan_paths, optarg, attr); - } -#endif + /* Taken in the first pass. */ break; case RA_OPT_CLOUDSYNC: #ifdef HAVE_CLOUDSYNC @@ -9259,8 +9207,8 @@ static bool retroarch_parse_input_and_config( } #ifdef HAVE_LIBRETRODB - if (scan_paths) - retroarch_cli_scan(scan_paths, explicit_menu, + if (scan_items) + retroarch_cli_scan(scan_items, scan_count, explicit_menu, (optind < argc) ? argv[optind] : NULL); #endif diff --git a/samples/tasks/database/database_scan_test.c b/samples/tasks/database/database_scan_test.c index e106487fd993..947426aaab89 100644 --- a/samples/tasks/database/database_scan_test.c +++ b/samples/tasks/database/database_scan_test.c @@ -503,7 +503,8 @@ bool __wrap_task_queue_is_threaded(void) return __real_task_queue_is_threaded(); } -/* A stream of slow_size bytes sleeps slow_ms before its first quantum; +/* A stream of slow_size bytes sleeps slow_ms before its first quantum, + * then waits while slow_hold is set, at most HOLD_USEC (below); * slow_in counts those sleeps begun, slow_back those ended (slow_begun * holds reader_begun as each of the first four ends), slow_by_scan * those in the scan's tick and slow_on_scan those on the thread that @@ -523,6 +524,7 @@ bool __wrap_task_queue_is_threaded(void) static uint32_t slow_size; static unsigned slow_ms; static retro_atomic_int_t slow_in; +static retro_atomic_int_t slow_hold; static retro_atomic_int_t slow_back; static int slow_begun[4]; static retro_atomic_int_t slow_on_scan; @@ -583,6 +585,7 @@ int64_t __wrap_intfstream_crc_step(intfstream_internal_t *intf, if (slow_size && size == (int64_t)slow_size && first) { int k; + retro_time_t until = cpu_features_get_time_usec() + HOLD_USEC; if (retro_atomic_load_acquire_size(&scan_thread_tid) == (size_t)current_tid()) @@ -591,6 +594,9 @@ int64_t __wrap_intfstream_crc_step(intfstream_internal_t *intf, retro_atomic_fetch_add_int(&slow_by_scan, 1); retro_atomic_fetch_add_int(&slow_in, 1); retro_sleep(slow_ms); + while ( retro_atomic_load_acquire_int(&slow_hold) + && cpu_features_get_time_usec() < until) + retro_sleep(1); if ((k = retro_atomic_fetch_add_int(&slow_back, 1)) < 4) slow_begun[k] = retro_atomic_load_acquire_int(&reader_begun); } @@ -9072,6 +9078,285 @@ static void core_lookup_lane(const char *root) ovr_cfg = NULL; } +/* What chain_lane()'s callbacks were handed; chain_in_push, what they + * were before task_push_dbscan_chain() returned. */ +#define CHAIN_SEEN_MAX 8 +static unsigned chain_seen; +static unsigned chain_dones; +static unsigned chain_in_push; +static bool chain_ud_ok; +static char chain_names[CHAIN_SEEN_MAX][512]; +static enum task_dbscan_outcome chain_outcomes[CHAIN_SEEN_MAX]; +static int chain_added[CHAIN_SEEN_MAX]; +static task_dbscan_chain_result_t chain_totals; + +static void chain_item_cb(const char *name, + enum task_dbscan_outcome outcome, + const task_dbscan_result_t *res, void *user_data) +{ + if (user_data != &chain_seen) + chain_ud_ok = false; + if (chain_seen < CHAIN_SEEN_MAX) + { + strlcpy(chain_names[chain_seen], name ? name : "(null)", + sizeof(chain_names[0])); + chain_outcomes[chain_seen] = outcome; + chain_added[chain_seen] = res ? (int)res->added : -1; + } + chain_seen++; +} + +static void chain_done_cb(const task_dbscan_chain_result_t *res, + void *user_data) +{ + if (user_data != &chain_seen) + chain_ud_ok = false; + chain_totals = *res; + chain_dones++; +} + +/* A --scan item for @arg; with @core, an override naming it. */ +static void chain_item(task_dbscan_item_t *it, const char *arg, + const char *core) +{ + it->kind = TASK_DBSCAN_ITEM_PATH; + it->arg = ovr_str(arg); + it->overrides = NULL; + if (core && (it->overrides = manual_content_scan_overrides_new())) + it->overrides->core = ovr_str(core); +} + +/* @n items, the first for @first, the rest for @rest. */ +static task_dbscan_item_t *chain_items(size_t n, const char *first, + const char *rest) +{ + size_t i; + task_dbscan_item_t *items = (task_dbscan_item_t*)calloc(n, + sizeof(*items)); + + if (items) + for (i = 0; i < n; i++) + chain_item(&items[i], i ? rest : first, NULL); + return items; +} + +static bool chain_push(task_dbscan_item_t *items, size_t n) +{ + bool pushed; + + chain_seen = 0; + chain_dones = 0; + chain_ud_ok = true; + memset(&chain_totals, 0, sizeof(chain_totals)); + pushed = task_push_dbscan_chain(items, n, chain_item_cb, + chain_done_cb, &chain_seen); + chain_in_push = chain_seen + chain_dones; + return pushed; +} + +/* Runs the queue until done_cb, then a while longer to see it is not + * called again. With @cancel_slow, cancels the scan once a slow read + * has begun, then lets a held one go. */ +static void chain_wait(bool cancel_slow) +{ + task_finder_data_t finder; + unsigned i; + bool cancelled = false; + time_t started = time(NULL); + + finder.func = find_scan; + finder.userdata = NULL; + while (!chain_dones) + { + if ( cancel_slow && !cancelled + && retro_atomic_load_acquire_int(&slow_in) > 0 + && task_queue_find(&finder)) + { + task_queue_cancel_ident(lane_ident); + retro_atomic_store_release_int(&slow_hold, 0); + cancelled = true; + } + task_queue_check(); + if (difftime(time(NULL), started) > SCAN_TIMEOUT_SECONDS) + return; + retro_sleep(1); + } + for (i = 0; i < 20; i++) + { + task_queue_check(); + retro_sleep(1); + } +} + +static bool chain_saw(unsigned i, const char *name, + enum task_dbscan_outcome outcome, int added) +{ + return i < chain_seen && i < CHAIN_SEEN_MAX + && !strcmp(chain_names[i], name) + && chain_outcomes[i] == outcome + && chain_added[i] == added; +} + +/* The chain runs --scan items in order, one task at a time: a missing + * path and an unknown core fail their item, not the run, and the two + * scans of one playlist follow each other, the second pushed from the + * first's callback. A cancelled scan stops the rest. A chain is + * refused while another runs, or a manual scan. */ +static void chain_lane(const char *root, const char *pl_dir, + uint32_t crc_a, uint32_t sz_a, bool threaded) +{ + char a[512], b[512], slow[512], missing[512], pl[512]; + char out[1024], err[1024], p[1024], what[96]; + char detail[160]; + const char *q = threaded ? "(threaded)" : "(regular)"; + task_dbscan_item_t *items; + bool pushed, refused, captured, back; + + sprintf(a, "%s/chain_a", root); + sprintf(b, "%s/chain_b", root); + sprintf(slow, "%s/chain_slow", root); + sprintf(missing, "%s/chain_missing", root); + sprintf(pl, "%s/chain_pl", root); + path_mkdir(a); + path_mkdir(b); + path_mkdir(slow); + path_mkdir(pl); + sprintf(p, "%s/01_alpha.bin", a); + if (!write_content(p, crc_a, sz_a)) + goto fixture; + sprintf(p, "%s/01_alpha.bin", b); + if (!write_content(p, crc_a, sz_a)) + goto fixture; + sprintf(p, "%s/00_slow.bin", slow); + if (!write_content(p, crc_a, sz_a)) + goto fixture; + sprintf(p, "%s/01_more.bin", slow); + if (!write_content(p, 0x3C0FFEE0u, 1024)) + goto fixture; + strlcpy(config_get_ptr()->paths.directory_playlist, pl, + sizeof(config_get_ptr()->paths.directory_playlist)); + delete_playlists(pl); + use_queue(threaded); + + pushed = chain_push(chain_items(1, missing, NULL), 1); + chain_wait(false); + sprintf(what, "chain %s: nothing inside the push", q); + sprintf(detail, "%u calls in the push, done %u", chain_in_push, + chain_dones); + check(pushed && !chain_in_push && chain_dones == 1, what, detail); + sprintf(what, "chain %s: a missing path fails", q); + check( chain_seen == 1 + && chain_saw(0, missing, TASK_DBSCAN_FAILED, -1) + && chain_totals.failed == 1 && !chain_totals.ran + && !chain_totals.added, what, chain_names[0]); + + sprintf(out, "%s/chain_out.txt", root); + sprintf(err, "%s/chain_err.txt", root); + items = (task_dbscan_item_t*)calloc(4, sizeof(*items)); + if (!items) + goto fixture; + chain_item(&items[0], a, NULL); + chain_item(&items[1], missing, NULL); + chain_item(&items[2], b, "nope"); + chain_item(&items[3], b, NULL); + cli_output = true; + captured = capture_begin(out, err); + pushed = chain_push(items, 4); + chain_wait(false); + if (captured) + capture_end(); + cli_output = false; + sprintf(what, "chain %s: items in order", q); + sprintf(detail, "%u items, done %u", chain_seen, chain_dones); + check( captured && pushed && chain_seen == 4 + && chain_saw(0, a, TASK_DBSCAN_RAN, 1) + && chain_saw(1, missing, TASK_DBSCAN_FAILED, -1) + && chain_saw(2, b, TASK_DBSCAN_FAILED, -1) + && chain_saw(3, b, TASK_DBSCAN_RAN, 1), what, detail); + sprintf(what, "chain %s: the totals", q); + sprintf(detail, "ran %u, added %u, failed %u, skipped %u%s", + chain_totals.ran, chain_totals.added, chain_totals.failed, + chain_totals.skipped, chain_totals.cancelled ? ", cancelled" : ""); + check( chain_totals.ran == 2 && chain_totals.added == 2 + && chain_totals.failed == 2 && !chain_totals.skipped + && !chain_totals.cancelled, what, detail); + sprintf(what, "chain %s: done_cb once, with the user data", q); + check(chain_dones == 1 && chain_ud_ok && !chain_in_push, what, ""); + sprintf(what, "chain %s: one playlist twice in a row", q); + sprintf(p, "%s/Test Alpha.lpl", pl); + { + char ea[1024], eb[1024]; + + sprintf(ea, "%s/01_alpha.bin", a); + sprintf(eb, "%s/01_alpha.bin", b); + check(file_contains(p, ea) && file_contains(p, eb), what, p); + } + sprintf(what, "chain %s: the item lines", q); + sprintf(p, "%s: not found", missing); + check( file_contains(err, p) + && file_contains(err, "nope: no such core"), what, err); + + /* Held on the threaded queue, so the scan cannot end before the + * cancel; the regular one reads on this thread */ + slow_reset(sz_a, 300); + if (threaded) + retro_atomic_store_release_int(&slow_hold, 1); + pushed = chain_push(chain_items(2, slow, a), 2); + chain_wait(true); + retro_atomic_store_release_int(&slow_hold, 0); + slow_size = 0; + back = slow_reads_back() && readers_gone(); + sprintf(what, "chain %s: a cancelled scan stops the rest", q); + sprintf(detail, "%u items, ran %u%s, done %u", chain_seen, + chain_totals.ran, chain_totals.cancelled ? ", cancelled" : "", + chain_dones); + check( pushed && back && chain_seen == 1 && chain_dones == 1 + && chain_saw(0, slow, TASK_DBSCAN_RAN, chain_added[0]) + && chain_totals.ran == 1 && chain_totals.cancelled, what, detail); + + pushed = chain_push(chain_items(1, a, NULL), 1); + refused = !task_push_dbscan_chain(chain_items(1, b, NULL), 1, + chain_item_cb, chain_done_cb, &chain_seen); + chain_wait(false); + sprintf(what, "chain %s: a second chain refused", q); + sprintf(detail, "%u items, done %u", chain_seen, chain_dones); + check( pushed && refused && chain_seen == 1 && chain_dones == 1 + && chain_saw(0, a, TASK_DBSCAN_RAN, 0), what, detail); + + /* Held, so the scan cannot end before the chain is pushed */ + slow_reset(sz_a, 0); + retro_atomic_store_release_int(&slow_hold, 1); + loop_active = true; + scan_completed = false; + pushed = task_push_dbscan(a, scan_cb, NULL); + refused = !chain_push(chain_items(1, b, NULL), 1); + retro_atomic_store_release_int(&slow_hold, 0); + { + time_t started = time(NULL); + + while (loop_active && difftime(time(NULL), started) + <= SCAN_TIMEOUT_SECONDS) + { + task_queue_check(); + retro_sleep(1); + } + } + slow_size = 0; + sprintf(what, "chain %s: refused while a manual scan runs", q); + check( pushed && refused && scan_completed && !chain_seen + && !chain_dones, what, refused ? "refused" : "pushed"); + + manual_content_scan_reset_settings(); + strlcpy(config_get_ptr()->paths.directory_playlist, pl_dir, + sizeof(config_get_ptr()->paths.directory_playlist)); + return; + +fixture: + check(0, "fixture", "could not write the chain lane"); + strlcpy(config_get_ptr()->paths.directory_playlist, pl_dir, + sizeof(config_get_ptr()->paths.directory_playlist)); +} + int main(int argc, char **argv) { const char *root = (argc > 1) ? argv[1] : "/tmp"; @@ -10157,6 +10442,8 @@ int main(int argc, char **argv) reset_lane(root, pl_dir); overrides_lane(root, pl_dir); core_lookup_lane(root); + chain_lane(root, pl_dir, crc_a, sz_a, false); + chain_lane(root, pl_dir, crc_a, sz_a, true); { char reader_db[512]; diff --git a/tasks/task_database.c b/tasks/task_database.c index 3a3567327fb6..7243e8bc18c2 100644 --- a/tasks/task_database.c +++ b/tasks/task_database.c @@ -5847,3 +5847,384 @@ bool task_push_manual_content_scan( return false; } + +/* --- The scan chain ----------------------------------------------- */ + +/* A prep task's state: the handler fills it, and the callback takes + * what it needs on the main thread once the task finished. */ +typedef struct +{ + char *dir; + dir_list_iter_t *iter; + struct string_list *list; + bool include_hidden; + bool begun; +} dbscan_prep_t; + +typedef struct +{ + task_dbscan_item_t *items; + struct string_list *rdb_list; + char *name; /* the running scan's */ + task_dbscan_item_cb_t item_cb; + task_dbscan_chain_cb_t done_cb; + void *user_data; + size_t count; + size_t next; + task_dbscan_chain_result_t totals; + bool cli_output; +} dbscan_chain_t; + +/* Main thread only, as every step of the chain. */ +static dbscan_chain_t *dbscan_chain_running = NULL; + +static void dbscan_chain_free_items(task_dbscan_item_t *items, + size_t count) +{ + size_t i; + + if (!items) + return; + for (i = 0; i < count; i++) + { + free(items[i].arg); + manual_content_scan_overrides_free(items[i].overrides); + } + free(items); +} + +static void dbscan_chain_finish(dbscan_chain_t *chain) +{ + if (chain->done_cb) + chain->done_cb(&chain->totals, chain->user_data); + dbscan_chain_free_items(chain->items, chain->count); + if (chain->rdb_list) + string_list_free(chain->rdb_list); + free(chain->name); + free(chain); + dbscan_chain_running = NULL; +} + +static void dbscan_chain_fold(dbscan_chain_t *chain, const char *name, + enum task_dbscan_outcome outcome, const task_dbscan_result_t *res) +{ + switch (outcome) + { + case TASK_DBSCAN_RAN: + chain->totals.ran++; + if (res && res->added) + chain->totals.added++; + break; + case TASK_DBSCAN_FAILED: + chain->totals.failed++; + break; + case TASK_DBSCAN_SKIPPED: + chain->totals.skipped++; + break; + } + if (chain->item_cb) + chain->item_cb(name, outcome, res, chain->user_data); +} + +/* @in with '~' expanded, made absolute if relative; NULL, with the + * reason on stderr, if nothing is there. Free the result. */ +static char *dbscan_chain_resolve(const char *in) +{ + char *s = (char*)malloc(PATH_MAX_LENGTH); + + if (!s) + { + fprintf(stderr, "out of memory\n"); + return NULL; + } + fill_pathname_expand_special(s, in, PATH_MAX_LENGTH); + /* "" would resolve to the working directory. Consoles resolve + * nothing, so a relative path stays as it is there. */ +#ifndef RARCH_CONSOLE + if ( *s && !path_is_absolute(s) + && !path_resolve_realpath(s, PATH_MAX_LENGTH, false)) + *s = '\0'; +#endif + if (!*s || !path_is_valid(s)) + { + fprintf(stderr, "%s: not found\n", in); + free(s); + return NULL; + } + return s; +} + +/* @name: the item, for the one error the overrides cannot name. */ +static void dbscan_chain_override_error(const dbscan_chain_t *chain, + const char *name, const manual_content_scan_overrides_t *ov, + enum manual_content_scan_override_status st, unsigned count) +{ + if (!chain->cli_output) + return; + switch (st) + { + case MANUAL_CONTENT_SCAN_OVR_NO_SUCH_CORE: + fprintf(stderr, "%s: no such core\n", ov->core); + break; + case MANUAL_CONTENT_SCAN_OVR_AMBIGUOUS_CORE: + fprintf(stderr, "%s: %u cores with this name, give the path\n", + ov->core, count); + break; + case MANUAL_CONTENT_SCAN_OVR_NO_SUCH_DATABASE: + fprintf(stderr, "%s: no such database\n", ov->database); + break; + case MANUAL_CONTENT_SCAN_OVR_NOT_A_DAT: + fprintf(stderr, "%s: not a DAT file\n", ov->dat); + break; + case MANUAL_CONTENT_SCAN_OVR_DAT_TOO_LARGE: + fprintf(stderr, "%s: DAT file too large\n", ov->dat); + break; + case MANUAL_CONTENT_SCAN_OVR_DAT_WITHOUT_MATCH: + fprintf(stderr, + "%s: --scan-dat cannot be used with --scan-match=none\n", + name); + break; + case MANUAL_CONTENT_SCAN_OVR_OK: + break; + } +} + +static void dbscan_chain_next(dbscan_chain_t *chain); + +static void dbscan_chain_scan_cb(retro_task_t *task, void *task_data, + void *user_data, const char *err) +{ + dbscan_chain_t *chain = (dbscan_chain_t*)user_data; + const task_dbscan_result_t *res = (const task_dbscan_result_t*)task_data; + char *name = chain->name; + + chain->name = NULL; + dbscan_chain_fold(chain, name, + res->failed ? TASK_DBSCAN_FAILED : TASK_DBSCAN_RAN, res); + free(name); + /* Cancelled, as on quit: start no more. */ + if (task_get_flags(task) & RETRO_TASK_FLG_CANCELLED) + { + chain->totals.cancelled = true; + dbscan_chain_finish(chain); + } + else + dbscan_chain_next(chain); +} + +/* --scan-overwrite, --scan-validate and --scan-omit-db-ref act on one + * playlist only: warns of those in @flags when the applied settings + * give none. */ +static void dbscan_chain_single_warn(const dbscan_chain_t *chain, + const char *name, unsigned flags) +{ + if (!chain->cli_output || manual_content_scan_is_single_playlist()) + return; + if (flags & MANUAL_CONTENT_SCAN_OVR_OVERWRITE) + fprintf(stderr, "%s: --scan-overwrite needs one playlist, ignored\n", + name); + if (flags & MANUAL_CONTENT_SCAN_OVR_VALIDATE) + fprintf(stderr, "%s: --scan-validate needs one playlist, ignored\n", + name); + if (flags & MANUAL_CONTENT_SCAN_OVR_OMIT_DB_REF) + fprintf(stderr, "%s: --scan-omit-db-ref needs one playlist, ignored\n", + name); +} + +/* True once the item's scan is pushed; otherwise it is folded as + * failed. */ +static bool dbscan_chain_push_path(dbscan_chain_t *chain, + const task_dbscan_item_t *item) +{ + enum manual_content_scan_override_status st; + unsigned count = 0; + char *path = dbscan_chain_resolve(item->arg); + + if (!path) + { + dbscan_chain_fold(chain, item->arg, TASK_DBSCAN_FAILED, NULL); + return false; + } + manual_content_scan_reset_settings(); + manual_content_scan_set_menu_content_dir(path); + if ((st = manual_content_scan_apply_overrides(item->overrides, + chain->rdb_list, &count)) != MANUAL_CONTENT_SCAN_OVR_OK) + dbscan_chain_override_error(chain, item->arg, item->overrides, st, + count); + else + { + unsigned flags = item->overrides ? item->overrides->flags : 0; + + if ( chain->cli_output + && (flags & MANUAL_CONTENT_SCAN_OVR_OVERWRITE) + && !path_is_directory(path)) + { + fprintf(stderr, "%s: --scan-overwrite ignored for a single file\n", + path); + flags &= ~(unsigned)MANUAL_CONTENT_SCAN_OVR_OVERWRITE; + } + dbscan_chain_single_warn(chain, path, flags); + if (task_push_manual_content_scan(false, dbscan_chain_scan_cb, chain)) + { + chain->name = path; + return true; + } + if (chain->cli_output) + fprintf(stderr, "%s: the scan could not start\n", path); + } + dbscan_chain_fold(chain, path, TASK_DBSCAN_FAILED, NULL); + free(path); + return false; +} + +/* Runs items until one pushes a task; after the last, finishes. */ +static void dbscan_chain_next(dbscan_chain_t *chain) +{ + while (chain->next < chain->count) + { + const task_dbscan_item_t *item = &chain->items[chain->next++]; + + switch (item->kind) + { + case TASK_DBSCAN_ITEM_PATH: + if (dbscan_chain_push_path(chain, item)) + return; + break; + default: + dbscan_chain_fold(chain, item->arg, TASK_DBSCAN_FAILED, NULL); + break; + } + } + dbscan_chain_finish(chain); +} + +/* Lists the database folder: to the end on a threaded queue's worker, + * else under the shared window. */ +static void dbscan_prep_handler(retro_task_t *task) +{ + dbscan_prep_t *prep = (dbscan_prep_t*)task->state; + int r = 1; + + if (task_get_flags(task) & RETRO_TASK_FLG_CANCELLED) + { + task_set_flags(task, RETRO_TASK_FLG_FINISHED, true); + return; + } + if (!prep->begun) + { + prep->begun = true; + if (prep->dir && (prep->list = string_list_new())) + prep->iter = dir_list_iter_new(prep->dir, "rdb", false, + prep->include_hidden, false, false, prep->list); + } + if (prep->iter) + { + if (task_queue_is_threaded()) + r = dir_list_iter_step(prep->iter, NULL, NULL); + else + { + nbio_budget_t b; + task_nbio_slice_open(&b); + r = dir_list_iter_step(prep->iter, + manual_scan_walk_within_budget, &b); + task_nbio_slice_close(&b); + if (!r) + return; + } + } + /* No listing, as dir_list_new() gives none. */ + if ((!prep->iter || r < 0) && prep->list) + { + string_list_free(prep->list); + prep->list = NULL; + } + task_set_flags(task, RETRO_TASK_FLG_FINISHED, true); +} + +static void dbscan_prep_cb(retro_task_t *task, void *task_data, + void *user_data, const char *err) +{ + dbscan_chain_t *chain = (dbscan_chain_t*)user_data; + dbscan_prep_t *prep = (dbscan_prep_t*)task->state; + + chain->rdb_list = prep->list; + prep->list = NULL; + if (task_get_flags(task) & RETRO_TASK_FLG_CANCELLED) + { + chain->totals.cancelled = true; + dbscan_chain_finish(chain); + } + else + dbscan_chain_next(chain); +} + +static void dbscan_prep_free(retro_task_t *task) +{ + dbscan_prep_t *prep = (dbscan_prep_t*)task->state; + + if (!prep) + return; + dir_list_iter_free(prep->iter); + if (prep->list) + string_list_free(prep->list); + free(prep->dir); + free(prep); +} + +/* Any unfinished scan, whatever its playlist. */ +static bool dbscan_chain_scan_finder(retro_task_t *task, void *user_data) +{ + return task->handler == task_manual_content_scan_handler + && !(task_get_flags(task) & RETRO_TASK_FLG_FINISHED); +} + +bool task_push_dbscan_chain(task_dbscan_item_t *items, size_t count, + task_dbscan_item_cb_t item_cb, task_dbscan_chain_cb_t done_cb, + void *user_data) +{ + task_finder_data_t find_data; + retro_task_t *task = NULL; + dbscan_chain_t *chain = NULL; + dbscan_prep_t *prep = NULL; + settings_t *settings = config_get_ptr(); + + find_data.func = dbscan_chain_scan_finder; + find_data.userdata = NULL; + if (dbscan_chain_running || task_queue_find(&find_data)) + goto error; + if ( !(chain = (dbscan_chain_t*)calloc(1, sizeof(*chain))) + || !(prep = (dbscan_prep_t*)calloc(1, sizeof(*prep))) + || !(task = task_init())) + goto error; +#ifdef HAVE_LIBRETRODB + if ( *settings->paths.path_content_database + && !(prep->dir = strdup(settings->paths.path_content_database))) + goto error; +#endif + prep->include_hidden = settings->bools.show_hidden_files; + + chain->items = items; + chain->count = count; + chain->item_cb = item_cb; + chain->done_cb = done_cb; + chain->user_data = user_data; + chain->cli_output = retroarch_override_setting_is_set( + RARCH_OVERRIDE_SETTING_DATABASE_SCAN, NULL); + + /* Silent: no title, no notifications. */ + task->handler = dbscan_prep_handler; + task->state = prep; + task->user_data = chain; + task->callback = dbscan_prep_cb; + task->cleanup = dbscan_prep_free; + dbscan_chain_running = chain; + task_queue_push(task); + return true; + +error: + free(task); + free(prep); + free(chain); + dbscan_chain_free_items(items, count); + return false; +} diff --git a/tasks/tasks_internal.h b/tasks/tasks_internal.h index d4aaa70e1ab2..ae5a9b9d33ae 100644 --- a/tasks/tasks_internal.h +++ b/tasks/tasks_internal.h @@ -356,6 +356,55 @@ bool task_push_dbscan(const char *fullpath, retro_task_callback_t cb, void *user_data); #endif +struct manual_content_scan_overrides; + +enum task_dbscan_item_kind +{ + TASK_DBSCAN_ITEM_PATH = 0, /* --scan */ + TASK_DBSCAN_ITEM_REFRESH, /* --scan-refresh */ + TASK_DBSCAN_ITEM_REFRESH_ALL /* --scan-refresh-all, Refresh All */ +}; + +typedef struct +{ + char *arg; /* as typed; NULL for REFRESH_ALL */ + struct manual_content_scan_overrides *overrides; /* NULL: none */ + enum task_dbscan_item_kind kind; +} task_dbscan_item_t; + +enum task_dbscan_outcome +{ + TASK_DBSCAN_RAN = 0, /* res holds the counts */ + TASK_DBSCAN_FAILED, + TASK_DBSCAN_SKIPPED +}; + +typedef struct +{ + unsigned ran; /* scans that ran and did not fail */ + unsigned added; /* of those, the ones that added an entry */ + unsigned failed; + unsigned skipped; + bool cancelled; +} task_dbscan_chain_result_t; + +typedef void (*task_dbscan_item_cb_t)(const char *name, + enum task_dbscan_outcome outcome, + const task_dbscan_result_t *res, void *user_data); +typedef void (*task_dbscan_chain_cb_t)( + const task_dbscan_chain_result_t *res, void *user_data); + +/* Runs @items in order, one task at a time: each step is pushed from + * the last one's callback, and a cancelled task ends the run. The + * chain owns @items from the call, whatever it returns. @item_cb: once + * per scan, failed item or skipped playlist; @done_cb: once at the end, + * from a task callback. @res is valid during the call only. False, + * calling neither, if out of memory or while a chain or a manual scan + * runs. Main thread only. */ +bool task_push_dbscan_chain(task_dbscan_item_t *items, size_t count, + task_dbscan_item_cb_t item_cb, task_dbscan_chain_cb_t done_cb, + void *user_data); + bool task_push_manual_content_scan( bool do_menu_refresh, retro_task_callback_t user_cb, void *user_data); From c855d737296ee9a6cca154ab1386440be30f3633 Mon Sep 17 00:00:00 2001 From: davey Date: Sat, 31 Oct 2026 19:38:08 -0700 Subject: [PATCH 13/21] retroarch: Manual Scan options for --scan --scan-playlist, --scan-database, --scan-match, --scan-dat, --scan-core, --scan-exts and the five switches set the Manual Scan settings of the --scan before them; a --scan without any stays Fully Automatic. Values are checked in the first pass, so an option before any --scan, an empty or bad value, or one too long for its setting ends the run with its reason on stderr and status 1, before the config loads and before anything is scanned. Cores, databases and DAT files are resolved when their scan runs, and one that does not resolve fails only that scan. --- retroarch.c | 224 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 223 insertions(+), 1 deletion(-) diff --git a/retroarch.c b/retroarch.c index b4953daafae4..9fa940ba1012 100644 --- a/retroarch.c +++ b/retroarch.c @@ -231,6 +231,7 @@ #include "core.h" #include "configuration.h" #include "list_special.h" +#include "manual_content_scan.h" #ifdef HAVE_CHEATS #include "cheat_manager.h" #endif @@ -390,6 +391,17 @@ enum RA_OPT_MAX_FRAMES_SCREENSHOT_PATH, RA_OPT_SET_SHADER, RA_OPT_DATABASE_SCAN, + RA_OPT_SCAN_PLAYLIST, + RA_OPT_SCAN_DATABASE, + RA_OPT_SCAN_MATCH, + RA_OPT_SCAN_DAT, + RA_OPT_SCAN_CORE, + RA_OPT_SCAN_EXTS, + RA_OPT_SCAN_NO_RECURSIVE, + RA_OPT_SCAN_NO_ARCHIVES, + RA_OPT_SCAN_OVERWRITE, + RA_OPT_SCAN_VALIDATE, + RA_OPT_SCAN_OMIT_DB_REF, RA_OPT_ACCESSIBILITY, RA_OPT_LOAD_MENU_ON_ERROR, RA_OPT_CLOUDSYNC, @@ -7927,9 +7939,52 @@ static void retroarch_print_help(const char *arg0) " how a particular subsystem wants content to be loaded.\n"); #ifdef HAVE_LIBRETRODB + fputs(buf, stdout); + buf[0] = '\0'; + _len = 0; + strlcpy_append(buf, sizeof(buf), &_len, " --scan=PATH|FILE " - "Import content from path.\n"); + "Import content from path. Repeatable.\n" + " " + " The options from --scan-playlist on apply\n" + " " + " to the --scan before them. Without any,\n" + " " + " the scan is Fully Automatic.\n"); + strlcpy_append(buf, sizeof(buf), &_len, + " --scan-playlist=NAME " + "Playlist to add to.\n" + " --scan-database=NAME|first " + "Database to match: one by name, or the first\n" + " " + " that matches.\n" + " --scan-match=MODE " + "Database check: strict, loose or none.\n" + " --scan-dat=FILE " + "Arcade DAT file to match instead, strict unless\n" + " " + " --scan-match=loose.\n"); + strlcpy_append(buf, sizeof(buf), &_len, + " --scan-core=CORE " + "Core: a path, a file name or a display name.\n" + " --scan-exts=EXTS " + "File extensions, separated by commas or spaces.\n" + " --scan-no-recursive " + "Do not scan subfolders.\n" + " --scan-no-archives " + "Do not look inside archives.\n"); + strlcpy_append(buf, sizeof(buf), &_len, + " --scan-overwrite " + "Replace the playlist instead of adding to it.\n" + " --scan-validate " + "Drop entries for missing or unsupported files.\n" + " --scan-omit-db-ref " + "Look up thumbnails by the playlist name.\n"); + + fputs(buf, stdout); + buf[0] = '\0'; + _len = 0; #endif #ifdef HAVE_CLOUDSYNC strlcpy_append(buf, sizeof(buf), &_len, @@ -8289,6 +8344,135 @@ static bool retroarch_cli_scan_add(task_dbscan_item_t **items, return true; } +/* Sets scan option @c, one of @opts, on the latest of the @count + * items; a bad one ends the run before the config loads. */ +static void retroarch_cli_scan_option(task_dbscan_item_t *items, + size_t count, const struct option *opts, int c, const char *value) +{ + manual_content_scan_overrides_t *ov = NULL; + const char *err = NULL; + char **field = NULL; + char *copy = NULL; + size_t max_len = PATH_MAX_LENGTH - 1; + + while (opts->val != c) + opts++; + + if (!count) + err = "must follow a --scan"; + else + { + if (!(ov = items[count - 1].overrides)) + ov = items[count - 1].overrides = + manual_content_scan_overrides_new(); + if (opts->has_arg) + copy = strdup(value); + if (!ov || (opts->has_arg && !copy)) + { + fprintf(stderr, "out of memory\n"); + exit(1); + } + + switch (c) + { + case RA_OPT_SCAN_PLAYLIST: + field = &ov->playlist; + /* .lpl.tmp fits a file name */ + max_len = NAME_MAX_LENGTH - 9; + break; + case RA_OPT_SCAN_DATABASE: + if (string_is_equal(copy, "first")) + { + free(ov->database); + ov->database = NULL; + ov->flags |= MANUAL_CONTENT_SCAN_OVR_FIRST_DB; + } + else + { + ov->flags &= ~MANUAL_CONTENT_SCAN_OVR_FIRST_DB; + field = &ov->database; + max_len = NAME_MAX_LENGTH - 1; + } + break; + case RA_OPT_SCAN_MATCH: + if (!*copy) + err = "needs a value"; + else if (string_is_equal(copy, "strict")) + ov->match = MANUAL_CONTENT_SCAN_USE_DB_STRICT; + else if (string_is_equal(copy, "loose")) + ov->match = MANUAL_CONTENT_SCAN_USE_DB_LOOSE; + else if (string_is_equal(copy, "none")) + ov->match = MANUAL_CONTENT_SCAN_USE_DB_NONE; + else + err = "must be strict, loose or none"; + ov->flags |= MANUAL_CONTENT_SCAN_OVR_MATCH; + break; + case RA_OPT_SCAN_DAT: + field = &ov->dat; + break; + case RA_OPT_SCAN_CORE: + field = &ov->core; + break; + case RA_OPT_SCAN_EXTS: + { + /* Commas become spaces, runs of spaces one, ends trimmed */ + char *out = copy; + const char *in; + + for (in = copy; *in; in++) + { + char ch = (*in == ',') ? ' ' : *in; + if (ch != ' ' || (out > copy && out[-1] != ' ')) + *out++ = ch; + } + if (out > copy && out[-1] == ' ') + out--; + *out = '\0'; + field = &ov->exts; + } + break; + case RA_OPT_SCAN_NO_RECURSIVE: + ov->flags |= MANUAL_CONTENT_SCAN_OVR_NO_RECURSIVE; + break; + case RA_OPT_SCAN_NO_ARCHIVES: + ov->flags |= MANUAL_CONTENT_SCAN_OVR_NO_ARCHIVES; + break; + case RA_OPT_SCAN_OVERWRITE: + ov->flags |= MANUAL_CONTENT_SCAN_OVR_OVERWRITE; + break; + case RA_OPT_SCAN_VALIDATE: + ov->flags |= MANUAL_CONTENT_SCAN_OVR_VALIDATE; + break; + case RA_OPT_SCAN_OMIT_DB_REF: + ov->flags |= MANUAL_CONTENT_SCAN_OVR_OMIT_DB_REF; + break; + } + + if (field) + { + if (!*copy) + err = "needs a value"; + else if (strlen(copy) > max_len) + err = "is too long"; + else + { + free(*field); + *field = copy; + copy = NULL; + } + } + } + + free(copy); + if (err) + { + /* Before any --scan, no console yet on Windows. */ + frontend_driver_attach_console(); + fprintf(stderr, "--%s %s\n", opts->name, err); + exit(1); + } +} + /* What the chain told a run without --menu. */ typedef struct { @@ -8490,6 +8674,17 @@ static bool retroarch_parse_input_and_config( { "entryslot", 1, NULL, 'e' }, #ifdef HAVE_LIBRETRODB { "scan", 1, NULL, RA_OPT_DATABASE_SCAN }, + { "scan-playlist", 1, NULL, RA_OPT_SCAN_PLAYLIST }, + { "scan-database", 1, NULL, RA_OPT_SCAN_DATABASE }, + { "scan-match", 1, NULL, RA_OPT_SCAN_MATCH }, + { "scan-dat", 1, NULL, RA_OPT_SCAN_DAT }, + { "scan-core", 1, NULL, RA_OPT_SCAN_CORE }, + { "scan-exts", 1, NULL, RA_OPT_SCAN_EXTS }, + { "scan-no-recursive", 0, NULL, RA_OPT_SCAN_NO_RECURSIVE }, + { "scan-no-archives", 0, NULL, RA_OPT_SCAN_NO_ARCHIVES }, + { "scan-overwrite", 0, NULL, RA_OPT_SCAN_OVERWRITE }, + { "scan-validate", 0, NULL, RA_OPT_SCAN_VALIDATE }, + { "scan-omit-db-ref", 0, NULL, RA_OPT_SCAN_OMIT_DB_REF }, #endif #ifdef HAVE_CLOUDSYNC { "cloudsync", 0, NULL, RA_OPT_CLOUDSYNC }, @@ -8694,6 +8889,22 @@ static bool retroarch_parse_input_and_config( fprintf(stderr, "out of memory\n"); exit(1); } +#endif + break; + case RA_OPT_SCAN_PLAYLIST: + case RA_OPT_SCAN_DATABASE: + case RA_OPT_SCAN_MATCH: + case RA_OPT_SCAN_DAT: + case RA_OPT_SCAN_CORE: + case RA_OPT_SCAN_EXTS: + case RA_OPT_SCAN_NO_RECURSIVE: + case RA_OPT_SCAN_NO_ARCHIVES: + case RA_OPT_SCAN_OVERWRITE: + case RA_OPT_SCAN_VALIDATE: + case RA_OPT_SCAN_OMIT_DB_REF: +#ifdef HAVE_LIBRETRODB + retroarch_cli_scan_option(scan_items, scan_count, opts, c, + optarg); #endif break; #if defined(HAVE_LIBRETRODB) && defined(HAVE_NETWORKING) && defined(HAVE_NETWORK_CMD) && defined(HAVE_COMMAND) @@ -9161,6 +9372,17 @@ static bool retroarch_parse_input_and_config( } break; case RA_OPT_DATABASE_SCAN: + case RA_OPT_SCAN_PLAYLIST: + case RA_OPT_SCAN_DATABASE: + case RA_OPT_SCAN_MATCH: + case RA_OPT_SCAN_DAT: + case RA_OPT_SCAN_CORE: + case RA_OPT_SCAN_EXTS: + case RA_OPT_SCAN_NO_RECURSIVE: + case RA_OPT_SCAN_NO_ARCHIVES: + case RA_OPT_SCAN_OVERWRITE: + case RA_OPT_SCAN_VALIDATE: + case RA_OPT_SCAN_OMIT_DB_REF: /* Taken in the first pass. */ break; case RA_OPT_CLOUDSYNC: From c35cb42ce88d18e1110ce19727620d4f3d567786 Mon Sep 17 00:00:00 2001 From: davey Date: Sat, 31 Oct 2026 20:04:57 -0700 Subject: [PATCH 14/21] scan: refresh playlists in the chain A refresh item rescans one playlist with its saved scan settings, as the menu's Refresh Playlist does, with the item's scan options laid over them and written back by the scan. A refresh-all item does the same for every playlist in the playlist folder that has saved settings, in the order Manage Playlists lists them, histories and favorites left out. A refresh item names its playlist by name or by a path in the playlist folder. A bare "name.lpl" is the playlist folder's file, and an absolute path is normalised before it is checked against the folder. The name is looked up in the folder's listing on every platform, the exact name first, then ignoring case, since macOS and FAT/exFAT cards ignore case; the refresh then takes the file's own name, so a database's playlist stays one. Two or more playlists that match only in other cases fail the item. The playlist folder walk and each playlist load run in prep tasks: on the worker with a threaded queue, under the shared I/O window without one, each pushed from the previous step's callback. A playlist whose saved content folder is gone is skipped rather than failed. A listing that runs out of memory fails the refresh-all; a playlist folder that is missing or cannot be read lists nothing, and the refresh-all ends with "no playlists with saved scan settings". A refresh's start line names its content folder; its totals and failure lines name the playlist. --- manual_content_scan.c | 36 +- manual_content_scan.h | 6 + samples/tasks/database/Makefile | 10 +- samples/tasks/database/database_scan_test.c | 594 +++++++++++++++++ tasks/task_database.c | 688 ++++++++++++++++++-- 5 files changed, 1273 insertions(+), 61 deletions(-) diff --git a/manual_content_scan.c b/manual_content_scan.c index b9d7672d55c1..fe3ab70c2fc6 100644 --- a/manual_content_scan.c +++ b/manual_content_scan.c @@ -704,8 +704,8 @@ bool manual_content_scan_set_menu_core_name( * Returns MANUAL_CONTENT_SCAN_PLAYLIST_REFRESH_OK * if playlist contains a valid scan record. */ enum manual_content_scan_playlist_refresh_status - manual_content_scan_set_menu_from_playlist(playlist_t *playlist, - const char *path_content_database, bool show_hidden_files) + manual_content_scan_set_menu_from_playlist_list(playlist_t *playlist, + const struct string_list *rdb_list) { const char *playlist_path = NULL; const char *content_dir = NULL; @@ -719,9 +719,6 @@ enum manual_content_scan_playlist_refresh_status bool overwrite_playlist = false; bool omit_db_ref = false; int db_usage = MANUAL_CONTENT_SCAN_USE_DB_NONE; -#ifdef HAVE_LIBRETRODB - struct string_list *rdb_list = NULL; -#endif enum manual_content_scan_system_name_type system_name_type = MANUAL_CONTENT_SCAN_SYSTEM_NAME_CONTENT_DIR; enum manual_content_scan_core_type @@ -776,13 +773,6 @@ enum manual_content_scan_playlist_refresh_status scan_settings.db_selection = MANUAL_CONTENT_SCAN_SELECT_DB_AUTO; /* Set system name */ -#ifdef HAVE_LIBRETRODB - /* > If platform has database support, get names - * of all installed database files */ - rdb_list = dir_list_new_special( - path_content_database, - DIR_LIST_DATABASES, NULL, show_hidden_files); - if (rdb_list && rdb_list->size) { size_t i; @@ -818,9 +808,6 @@ enum manual_content_scan_playlist_refresh_status } } - string_list_free(rdb_list); -#endif - /* > If system name does not match a database * file, then check whether it matches the * content directory name */ @@ -912,6 +899,25 @@ enum manual_content_scan_playlist_refresh_status return MANUAL_CONTENT_SCAN_PLAYLIST_REFRESH_OK; } +enum manual_content_scan_playlist_refresh_status + manual_content_scan_set_menu_from_playlist(playlist_t *playlist, + const char *path_content_database, bool show_hidden_files) +{ + enum manual_content_scan_playlist_refresh_status status; + struct string_list *rdb_list = NULL; +#ifdef HAVE_LIBRETRODB + /* > If platform has database support, get names + * of all installed database files */ + rdb_list = dir_list_new_special( + path_content_database, + DIR_LIST_DATABASES, NULL, show_hidden_files); +#endif + status = manual_content_scan_set_menu_from_playlist_list(playlist, + rdb_list); + string_list_free(rdb_list); + return status; +} + /* Whether @name is an installed database: the basename, * without extension, of a path in @rdb_list */ static bool manual_content_scan_rdb_list_has( diff --git a/manual_content_scan.h b/manual_content_scan.h index 776a9af73134..8e8614467981 100644 --- a/manual_content_scan.h +++ b/manual_content_scan.h @@ -309,6 +309,12 @@ enum manual_content_scan_playlist_refresh_status manual_content_scan_set_menu_from_playlist(playlist_t *playlist, const char *path_content_database, bool show_hidden_files); +/* As manual_content_scan_set_menu_from_playlist(), with the + * paths of the installed databases in @rdb_list (NULL: none). */ +enum manual_content_scan_playlist_refresh_status + manual_content_scan_set_menu_from_playlist_list(playlist_t *playlist, + const struct string_list *rdb_list); + manual_content_scan_overrides_t *manual_content_scan_overrides_new(void); void manual_content_scan_overrides_free(manual_content_scan_overrides_t *ov); diff --git a/samples/tasks/database/Makefile b/samples/tasks/database/Makefile index 747951601fb1..b90d13ce9b8e 100644 --- a/samples/tasks/database/Makefile +++ b/samples/tasks/database/Makefile @@ -249,8 +249,10 @@ $(INDEX_TEST)$(EXE_EXT): $(INDEX_TEST_OBJECTS) # count and refuse reader pushes, count queue worker starts, count an # archive's stored-CRC and directory reads, fail a chosen serial's # query, order the database and content lists, count reader passes, -# hold chosen lookups, count an archive's member hits a step and hold -# a chosen index build in one step. +# hold chosen lookups, count an archive's member hits a step, hold +# a chosen index build in one step, tell which thread the refresh +# walks and playlist loads ran on, fail a chosen task_init and fail the +# playlist folder walk. SCAN_TEST_WRAPS := -Wl,--wrap=libretrodb_open -Wl,--wrap=task_queue_park \ -Wl,--wrap=intfstream_crc_step -Wl,--wrap=rm3u_is_m3u \ -Wl,--wrap=task_nbio_slice_open -Wl,--wrap=task_nbio_slice_close \ @@ -264,7 +266,9 @@ SCAN_TEST_WRAPS := -Wl,--wrap=libretrodb_open -Wl,--wrap=task_queue_park \ -Wl,--wrap=database_info_dir_init_from_list \ -Wl,--wrap=database_info_index_job_step \ -Wl,--wrap=task_nbio_slice_within_budget \ - -Wl,--wrap=fill_pathname_join_delim + -Wl,--wrap=fill_pathname_join_delim \ + -Wl,--wrap=dir_list_iter_new -Wl,--wrap=dir_list_iter_step \ + -Wl,--wrap=playlist_parse_begin -Wl,--wrap=task_init ifeq ($(HAVE_THREADS), 1) SCAN_TEST_WRAPS += -Wl,--wrap=sthread_create endif diff --git a/samples/tasks/database/database_scan_test.c b/samples/tasks/database/database_scan_test.c index 947426aaab89..2a45c4de49f1 100644 --- a/samples/tasks/database/database_scan_test.c +++ b/samples/tasks/database/database_scan_test.c @@ -994,13 +994,36 @@ static retro_time_t push_cancel_usec; static retro_atomic_int_t pushes_tried; static retro_atomic_int_t pushes_ok; +/* The chain's untitled steps: the prep_cancel_at'th pushed (from 1) is + * cancelled once its playlist-folder walk or playlist load begins. + * Set only between runs. */ +static unsigned prep_cancel_at; +static unsigned prep_pushes; +static retro_task_t *prep_cancel_task; + +static void prep_cancel_begun(void) +{ + retro_task_t *task = prep_cancel_task; + + if (task) + { + prep_cancel_task = NULL; + task_set_flags(task, RETRO_TASK_FLG_CANCELLED, true); + } +} + bool __real_task_queue_push(retro_task_t *task); bool __wrap_task_queue_push(retro_task_t *task) { int n; if (!(task_get_flags(task) & RETRO_TASK_FLG_PARALLEL)) + { + if ( prep_cancel_at && !task->title + && ++prep_pushes == prep_cancel_at) + prep_cancel_task = task; return __real_task_queue_push(task); + } n = retro_atomic_fetch_add_int(&pushes_tried, 1) + 1; if (push_fail_from && (unsigned)n >= push_fail_from) return false; @@ -9357,6 +9380,575 @@ static void chain_lane(const char *root, const char *pl_dir, sizeof(config_get_ptr()->paths.directory_playlist)); } +/* Linked with --wrap: how many database listings, playlist-folder + * walks and playlist loads began, and how many of them on the main + * thread. */ +enum { REFRESH_RDB = 0, REFRESH_LPL, REFRESH_LOAD, REFRESH_KINDS }; +static retro_atomic_int_t refresh_began[REFRESH_KINDS]; +static retro_atomic_int_t refresh_began_main[REFRESH_KINDS]; + +static void refresh_began_reset(void) +{ + int k; + + for (k = 0; k < REFRESH_KINDS; k++) + { + retro_atomic_store_release_int(&refresh_began[k], 0); + retro_atomic_store_release_int(&refresh_began_main[k], 0); + } +} + +static void refresh_began_one(int k) +{ + retro_atomic_fetch_add_int(&refresh_began[k], 1); + if (task_is_on_main_thread()) + retro_atomic_fetch_add_int(&refresh_began_main[k], 1); +} + +/* Set by the walk's own thread, which is the one that steps it. */ +static retro_atomic_int_t lpl_walk_fail; +static dir_list_iter_t *lpl_fail_iter; + +dir_list_iter_t *__real_dir_list_iter_new(const char *dir, const char *ext, + bool include_dirs, bool include_hidden, bool include_compressed, + bool recursive, struct string_list *list); +dir_list_iter_t *__wrap_dir_list_iter_new(const char *dir, const char *ext, + bool include_dirs, bool include_hidden, bool include_compressed, + bool recursive, struct string_list *list) +{ + dir_list_iter_t *iter; + if (ext && !strcmp(ext, "rdb")) + refresh_began_one(REFRESH_RDB); + else if (ext && !strcmp(ext, "lpl")) + { + refresh_began_one(REFRESH_LPL); + prep_cancel_begun(); + } + iter = __real_dir_list_iter_new(dir, ext, include_dirs, include_hidden, + include_compressed, recursive, list); + if ( ext && !strcmp(ext, "lpl") + && retro_atomic_load_acquire_int(&lpl_walk_fail)) + lpl_fail_iter = iter; + return iter; +} + +/* Linked with --wrap: the playlist folder walk fails as out of + * memory does. */ +int __real_dir_list_iter_step(dir_list_iter_t *iter, + bool (*within_budget)(void *ud), void *ud); +int __wrap_dir_list_iter_step(dir_list_iter_t *iter, + bool (*within_budget)(void *ud), void *ud) +{ + if (iter && iter == lpl_fail_iter) + return -1; + return __real_dir_list_iter_step(iter, within_budget, ud); +} + +playlist_parse_t *__real_playlist_parse_begin(const playlist_config_t *config); +playlist_parse_t *__wrap_playlist_parse_begin(const playlist_config_t *config) +{ + refresh_began_one(REFRESH_LOAD); + prep_cancel_begun(); + return __real_playlist_parse_begin(config); +} + +/* Linked with --wrap: the task_init() this counts down to fails, as + * out of memory. */ +static retro_atomic_int_t task_init_fail_in; + +retro_task_t *__real_task_init(void); +retro_task_t *__wrap_task_init(void) +{ + if ( retro_atomic_load_acquire_int(&task_init_fail_in) > 0 + && retro_atomic_fetch_add_int(&task_init_fail_in, -1) == 1) + return NULL; + return __real_task_init(); +} + +/* Writes @lpl with one entry, @entry, and unless @dir is NULL a scan + * record for @dir with the saved core @core (NULL: detect). */ +static bool refresh_write(const char *lpl, const char *dir, + const char *core, const char *entry) +{ + playlist_config_t *cfg = (playlist_config_t*)calloc(1, sizeof(*cfg)); + playlist_t *pl = NULL; + struct playlist_entry e; + bool ok = false; + + if (!cfg) + return false; + playlist_config_set_path(cfg, lpl); + cfg->capacity = COLLECTION_SIZE; + if ((pl = playlist_init(cfg))) + { + memset(&e, 0, sizeof(e)); + e.path = (char*)entry; + e.label = (char*)"Stale"; + e.core_path = (char*)"DETECT"; + e.core_name = (char*)"DETECT"; + ok = playlist_push(pl, &e); + if (dir) + { + playlist_set_scan_content_dir(pl, dir); + playlist_set_scan_search_recursively(pl, true); + if (core) + playlist_set_default_core_name(pl, core); + } + playlist_write_file(pl); + playlist_free(pl); + ok = ok && path_is_valid(lpl); + } + free(cfg); + return ok; +} + +static void refresh_item(task_dbscan_item_t *it, + enum task_dbscan_item_kind kind, const char *arg) +{ + it->kind = kind; + it->arg = arg ? ovr_str(arg) : NULL; + it->overrides = NULL; +} + +/* Runs @items with stdout in @out and stderr in @err; @cli as for a + * --scan run. */ +static bool refresh_run(task_dbscan_item_t *items, size_t n, + const char *out, const char *err, bool cli) +{ + bool captured, pushed; + + cli_output = cli; + captured = capture_begin(out, err); + pushed = chain_push(items, n); + chain_wait(false); + if (captured) + capture_end(); + cli_output = false; + return captured && pushed && chain_dones == 1; +} + +/* True if @dir lists @name in that case; @detail gets what it lists. */ +static bool listed_as(const char *dir, const char *name, char *detail, + size_t size) +{ + size_t i; + bool found = false; + struct string_list *l = dir_list_new(dir, "lpl", false, false, + false, false); + + *detail = '\0'; + for (i = 0; l && i < l->size; i++) + { + const char *base = path_basename(l->elems[i].data); + + if (i) + strlcat(detail, " ", size); + strlcat(detail, base, size); + if (string_is_equal(base, name)) + found = true; + } + string_list_free(l); + return found; +} + +/* Refresh items over a playlist folder: a refresh-all takes every + * playlist with saved settings in name order, history and favorites + * never, skips one whose folder is gone and fails one whose core is; + * it refreshes a playlist an earlier item wrote. A refresh names its + * playlist by name or by a path in the playlist folder. The walks and + * loads run on the worker when the queue has one. */ +static void refresh_lane(const char *root, const char *pl_dir, + uint32_t crc_a, uint32_t sz_a, bool threaded) +{ + char pl[512], none_pl[512], a[512], gone[512]; + char fresh[512], other[512], case_pl[512]; + char dup_pl[512]; + char out[1024], err[1024]; + char p[1024], stale[1024], want[1536], what[128]; + char detail[160]; + const char *q = threaded ? "(threaded)" : "(regular)"; + task_dbscan_item_t *items; + bool ran, off_main = false; + +#ifdef HAVE_THREADS + off_main = threaded; +#endif + sprintf(pl, "%s/refresh_pl", root); + sprintf(none_pl, "%s/refresh_none", root); + sprintf(a, "%s/refresh_a", root); + sprintf(gone, "%s/refresh_gone", root); + sprintf(fresh, "%s/refresh_new", root); + sprintf(other, "%s/refresh_other", root); + sprintf(case_pl, "%s/refresh_case", root); + sprintf(dup_pl, "%s/refresh_dup", root); + sprintf(out, "%s/refresh_out.txt", root); + sprintf(err, "%s/refresh_err.txt", root); + sprintf(stale, "%s/00_gone.bin", a); + path_mkdir(pl); + path_mkdir(none_pl); + path_mkdir(a); + path_mkdir(fresh); + path_mkdir(other); + path_mkdir(case_pl); + path_mkdir(dup_pl); + delete_playlists(pl); + delete_playlists(none_pl); + delete_playlists(other); + delete_playlists(case_pl); + delete_playlists(dup_pl); + sprintf(p, "%s/01_alpha.bin", a); + if (!write_content(p, crc_a, sz_a)) + goto fixture; + sprintf(p, "%s/01_alpha.bin", fresh); + if (!write_content(p, crc_a, sz_a)) + goto fixture; + sprintf(p, "%s/a_ok.lpl", pl); + if (!refresh_write(p, a, NULL, stale)) + goto fixture; + sprintf(p, "%s/b_gone.lpl", pl); + if (!refresh_write(p, gone, NULL, stale)) + goto fixture; + sprintf(p, "%s/c_none.lpl", pl); + if (!refresh_write(p, NULL, NULL, stale)) + goto fixture; + sprintf(p, "%s/e_core.lpl", pl); + if (!refresh_write(p, a, "No Such Core", stale)) + goto fixture; + sprintf(p, "%s/x_history.lpl", pl); + if (!refresh_write(p, a, NULL, stale)) + goto fixture; + sprintf(p, "%s/content_favorites.lpl", pl); + if (!refresh_write(p, a, NULL, stale)) + goto fixture; + sprintf(p, "%s/c_none.lpl", none_pl); + if (!refresh_write(p, NULL, NULL, stale)) + goto fixture; + sprintf(p, "%s/x_history.lpl", none_pl); + if (!refresh_write(p, a, NULL, stale)) + goto fixture; + sprintf(p, "%s/a_ok.lpl", other); + if (!refresh_write(p, a, NULL, stale)) + goto fixture; + strlcpy(config_get_ptr()->paths.directory_playlist, pl, + sizeof(config_get_ptr()->paths.directory_playlist)); + use_queue(threaded); + + /* A --scan writing n_new, then the refresh-all. */ + if (!(items = (task_dbscan_item_t*)calloc(2, sizeof(*items)))) + goto fixture; + chain_item(&items[0], fresh, NULL); + if ((items[0].overrides = manual_content_scan_overrides_new())) + items[0].overrides->playlist = ovr_str("n_new"); + refresh_item(&items[1], TASK_DBSCAN_ITEM_REFRESH_ALL, NULL); + refresh_began_reset(); + ran = refresh_run(items, 2, out, err, true); + sprintf(what, "refresh %s: refresh-all in name order", q); + snprintf(detail, sizeof(detail), "%u items: %s, %s, %s, %s", chain_seen, + chain_names[1], chain_names[2], chain_names[3], chain_names[4]); + check( ran && chain_seen == 5 + && chain_saw(0, fresh, TASK_DBSCAN_RAN, 1) + && chain_saw(1, "a_ok", TASK_DBSCAN_RAN, 1) + && chain_saw(2, "b_gone", TASK_DBSCAN_SKIPPED, -1) + && chain_saw(3, "e_core", TASK_DBSCAN_FAILED, -1) + && chain_saw(4, "n_new", TASK_DBSCAN_RAN, 0), what, detail); + sprintf(what, "refresh %s: the totals", q); + sprintf(detail, "ran %u, added %u, failed %u, skipped %u%s", + chain_totals.ran, chain_totals.added, chain_totals.failed, + chain_totals.skipped, chain_totals.cancelled ? ", cancelled" : ""); + check( chain_totals.ran == 3 && chain_totals.added == 2 + && chain_totals.failed == 1 && chain_totals.skipped == 1 + && !chain_totals.cancelled, what, detail); + sprintf(what, "refresh %s: a stale entry removed", q); + sprintf(p, "%s/a_ok.lpl", pl); + sprintf(want, "%s/01_alpha.bin", a); + check(file_contains(p, want) && !file_contains(p, stale), what, p); + sprintf(what, "refresh %s: history and favorites never loaded", q); + sprintf(p, "%s/x_history.lpl", pl); + sprintf(detail, "%d loads", + retro_atomic_load_acquire_int(&refresh_began[REFRESH_LOAD])); + check( file_contains(p, stale) && !file_contains(p, want) + && retro_atomic_load_acquire_int(&refresh_began[REFRESH_LOAD]) == 5, + what, detail); + sprintf(what, "refresh %s: the refresh lines", q); + sprintf(want, "%s: skipped: content folder not found (%s)", "b_gone", + gone); + check( file_contains(out, + "a_ok: 1 added, 0 duplicates, 0 unmatched, 0 errors") + && file_contains(err, want) + && file_contains(err, "e_core: saved core not found (No Such Core)") + && !file_contains(out, "b_gone") + && !file_contains(err, "no playlists"), what, err); + sprintf(what, "refresh %s: walks and loads %s", q, + off_main ? "off the main thread" : "ran"); + sprintf(detail, "rdb %d (%d main), lpl %d (%d main), loads %d (%d main)", + retro_atomic_load_acquire_int(&refresh_began[REFRESH_RDB]), + retro_atomic_load_acquire_int(&refresh_began_main[REFRESH_RDB]), + retro_atomic_load_acquire_int(&refresh_began[REFRESH_LPL]), + retro_atomic_load_acquire_int(&refresh_began_main[REFRESH_LPL]), + retro_atomic_load_acquire_int(&refresh_began[REFRESH_LOAD]), + retro_atomic_load_acquire_int(&refresh_began_main[REFRESH_LOAD])); + check( retro_atomic_load_acquire_int(&refresh_began[REFRESH_RDB]) >= 1 + && retro_atomic_load_acquire_int(&refresh_began[REFRESH_LPL]) == 1 + && (!off_main || ( + !retro_atomic_load_acquire_int(&refresh_began_main[REFRESH_RDB]) + && !retro_atomic_load_acquire_int(&refresh_began_main[REFRESH_LPL]) + && !retro_atomic_load_acquire_int( + &refresh_began_main[REFRESH_LOAD]))), what, detail); + + /* By name with an override, by path, and the naming failures. */ + if (!(items = (task_dbscan_item_t*)calloc(5, sizeof(*items)))) + goto fixture; + refresh_item(&items[0], TASK_DBSCAN_ITEM_REFRESH, "a_ok"); + if ((items[0].overrides = manual_content_scan_overrides_new())) + items[0].overrides->exts = ovr_str("bin"); + sprintf(p, "%s/a_ok.lpl", pl); + refresh_item(&items[1], TASK_DBSCAN_ITEM_REFRESH, p); + sprintf(p, "%s/a_ok.lpl", other); + refresh_item(&items[2], TASK_DBSCAN_ITEM_REFRESH, p); + refresh_item(&items[3], TASK_DBSCAN_ITEM_REFRESH, "nope"); + refresh_item(&items[4], TASK_DBSCAN_ITEM_REFRESH, "c_none"); + ran = refresh_run(items, 5, out, err, true); + sprintf(what, "refresh %s: by name, by path, and what fails", q); + sprintf(detail, "%u items, ran %u, failed %u", chain_seen, + chain_totals.ran, chain_totals.failed); + check( ran && chain_seen == 5 + && chain_saw(0, "a_ok", TASK_DBSCAN_RAN, 0) + && chain_saw(1, "a_ok", TASK_DBSCAN_RAN, 0) + && chain_saw(2, p, TASK_DBSCAN_FAILED, -1) + && chain_saw(3, "nope", TASK_DBSCAN_FAILED, -1) + && chain_saw(4, "c_none", TASK_DBSCAN_FAILED, -1) + && chain_totals.ran == 2 && chain_totals.failed == 3 + && !chain_totals.added, what, detail); + sprintf(what, "refresh %s: an override written back", q); + sprintf(want, "%s/a_ok.lpl", pl); + check(file_contains(want, "\"scan_file_exts\": \"bin\""), what, want); + sprintf(what, "refresh %s: the naming lines", q); + sprintf(want, "%s: not in the playlist folder (%s)", p, pl); + check( file_contains(err, want) + && file_contains(err, "nope: no such playlist") + && file_contains(err, "c_none: no saved scan settings"), what, err); + + /* A file name without a folder is the playlist folder's. */ + if (!(items = (task_dbscan_item_t*)calloc(2, sizeof(*items)))) + goto fixture; + refresh_item(&items[0], TASK_DBSCAN_ITEM_REFRESH, "a_ok.lpl"); + refresh_item(&items[1], TASK_DBSCAN_ITEM_REFRESH, "A_OK.LPL"); + ran = refresh_run(items, 2, out, err, true); + sprintf(what, "refresh %s: a file name, not a path", q); + check( ran && chain_saw(0, "a_ok", TASK_DBSCAN_RAN, 0) + && chain_saw(1, "a_ok", TASK_DBSCAN_RAN, 0) + && path_get_size(err) == 0, what, err); + +#ifdef _WIN32 + /* All '\' and '\' before the file name only. */ + if (!(items = (task_dbscan_item_t*)calloc(2, sizeof(*items)))) + goto fixture; + sprintf(p, "%s/a_ok.lpl", pl); + pathname_conform_slashes_to_os(p); + refresh_item(&items[0], TASK_DBSCAN_ITEM_REFRESH, p); + sprintf(p, "%s\\a_ok.lpl", pl); + refresh_item(&items[1], TASK_DBSCAN_ITEM_REFRESH, p); + ran = refresh_run(items, 2, out, err, true); + sprintf(what, "refresh %s: a path with either slash", q); + sprintf(detail, "%u items, ran %u, failed %u", chain_seen, + chain_totals.ran, chain_totals.failed); + check( ran && chain_seen == 2 + && chain_saw(0, "a_ok", TASK_DBSCAN_RAN, 0) + && chain_saw(1, "a_ok", TASK_DBSCAN_RAN, 0) + && chain_totals.ran == 2 && !chain_totals.failed, what, detail); + + /* A playlist folder with '\', a path with '/'. */ + strlcpy(p, pl, sizeof(p)); + pathname_conform_slashes_to_os(p); + strlcpy(config_get_ptr()->paths.directory_playlist, p, + sizeof(config_get_ptr()->paths.directory_playlist)); + if (!(items = (task_dbscan_item_t*)calloc(2, sizeof(*items)))) + goto fixture; + refresh_item(&items[0], TASK_DBSCAN_ITEM_REFRESH, "a_ok"); + sprintf(p, "%s/a_ok.lpl", pl); + refresh_item(&items[1], TASK_DBSCAN_ITEM_REFRESH, p); + ran = refresh_run(items, 2, out, err, true); + strlcpy(config_get_ptr()->paths.directory_playlist, pl, + sizeof(config_get_ptr()->paths.directory_playlist)); + sprintf(what, "refresh %s: a playlist folder with '\\'", q); + sprintf(detail, "%u items, ran %u, failed %u", chain_seen, + chain_totals.ran, chain_totals.failed); + check( ran && chain_seen == 2 + && chain_saw(0, "a_ok", TASK_DBSCAN_RAN, 0) + && chain_saw(1, "a_ok", TASK_DBSCAN_RAN, 0) + && chain_totals.ran == 2 && !chain_totals.failed, what, detail); +#endif + + /* Another case: the lines, the system name and the save take the + * file's own name, so a database's playlist stays one. */ + sprintf(p, "%s/Test Beta.lpl", case_pl); + if (!refresh_write(p, a, NULL, stale)) + goto fixture; + strlcpy(config_get_ptr()->paths.directory_playlist, case_pl, + sizeof(config_get_ptr()->paths.directory_playlist)); + if (!(items = (task_dbscan_item_t*)calloc(2, sizeof(*items)))) + goto fixture; + refresh_item(&items[0], TASK_DBSCAN_ITEM_REFRESH, "test beta"); + sprintf(p, "%s/TEST BETA.LPL", case_pl); + refresh_item(&items[1], TASK_DBSCAN_ITEM_REFRESH, p); + ran = refresh_run(items, 2, out, err, true); + strlcpy(config_get_ptr()->paths.directory_playlist, pl, + sizeof(config_get_ptr()->paths.directory_playlist)); + sprintf(what, "refresh %s: another case, named as the file", q); + snprintf(detail, sizeof(detail), "%u items: %s, %s", chain_seen, + chain_names[0], chain_names[1]); + check( ran && chain_seen == 2 + && !strcmp(chain_names[0], "Test Beta") + && !strcmp(chain_names[1], "Test Beta") + && chain_outcomes[0] == TASK_DBSCAN_RAN + && chain_outcomes[1] == TASK_DBSCAN_RAN + && file_contains(out, "Test Beta: ") + && !file_contains(out, "test beta") + && !file_contains(out, "TEST BETA"), what, detail); + sprintf(what, "refresh %s: another case, a database's playlist", q); + check( manual_content_scan_get_menu_system_name_type() + == MANUAL_CONTENT_SCAN_SYSTEM_NAME_DATABASE, what, ""); + sprintf(what, "refresh %s: another case keeps the file's name", q); + check( listed_as(case_pl, "Test Beta.lpl", detail, sizeof(detail)) + && !strcmp(detail, "Test Beta.lpl"), what, detail); + + /* Two files in other cases: neither is the name; the exact one is. */ + sprintf(p, "%s/Test Beta.lpl", dup_pl); + if (!refresh_write(p, a, NULL, stale)) + goto fixture; + sprintf(p, "%s/TEST BETA.lpl", dup_pl); + /* Where case is ignored this is the same file, already written. */ + refresh_write(p, a, NULL, stale); + if ( listed_as(dup_pl, "Test Beta.lpl", detail, sizeof(detail)) + && listed_as(dup_pl, "TEST BETA.lpl", detail, sizeof(detail))) + { + strlcpy(config_get_ptr()->paths.directory_playlist, dup_pl, + sizeof(config_get_ptr()->paths.directory_playlist)); + if (!(items = (task_dbscan_item_t*)calloc(2, sizeof(*items)))) + goto fixture; + refresh_item(&items[0], TASK_DBSCAN_ITEM_REFRESH, "test beta"); + refresh_item(&items[1], TASK_DBSCAN_ITEM_REFRESH, "Test Beta"); + ran = refresh_run(items, 2, out, err, true); + strlcpy(config_get_ptr()->paths.directory_playlist, pl, + sizeof(config_get_ptr()->paths.directory_playlist)); + sprintf(what, "refresh %s: two in other cases, ambiguous", q); + check( ran && chain_saw(0, "test beta", TASK_DBSCAN_FAILED, -1) + && file_contains(err, + "test beta: 2 playlists with this name in other cases, " + "give the exact name\n"), + what, err); + sprintf(what, "refresh %s: two in other cases, the exact one", q); + snprintf(detail, sizeof(detail), "%u items, ran %u, failed %u", + chain_seen, chain_totals.ran, chain_totals.failed); + check( ran && chain_seen == 2 + && !strcmp(chain_names[1], "Test Beta") + && chain_outcomes[1] == TASK_DBSCAN_RAN + && chain_totals.ran == 1 && chain_totals.failed == 1, + what, detail); + } + else + printf(" skip refresh %s: two in other cases (the file system " + "ignores case: %s)\n", q, detail); + + /* Nothing to refresh. */ + strlcpy(config_get_ptr()->paths.directory_playlist, none_pl, + sizeof(config_get_ptr()->paths.directory_playlist)); + if (!(items = (task_dbscan_item_t*)calloc(1, sizeof(*items)))) + goto fixture; + refresh_item(&items[0], TASK_DBSCAN_ITEM_REFRESH_ALL, NULL); + ran = refresh_run(items, 1, out, err, true); + sprintf(what, "refresh %s: a refresh-all with nothing to refresh", q); + check( ran && !chain_seen && !chain_totals.ran && !chain_totals.failed + && !chain_totals.skipped + && file_contains(err, "no playlists with saved scan settings"), + what, err); + + /* The second task_init is the refresh-all's listing. */ + if (!(items = (task_dbscan_item_t*)calloc(1, sizeof(*items)))) + goto fixture; + refresh_item(&items[0], TASK_DBSCAN_ITEM_REFRESH_ALL, NULL); + retro_atomic_store_release_int(&task_init_fail_in, 2); + ran = refresh_run(items, 1, out, err, true); + retro_atomic_store_release_int(&task_init_fail_in, 0); + sprintf(what, "refresh %s: a refresh-all out of memory", q); + check( ran && chain_saw(0, none_pl, TASK_DBSCAN_FAILED, -1) + && chain_totals.failed == 1 && !chain_totals.ran + && file_contains(err, "out of memory"), what, err); + + /* The listing fails part way. */ + if (!(items = (task_dbscan_item_t*)calloc(1, sizeof(*items)))) + goto fixture; + refresh_item(&items[0], TASK_DBSCAN_ITEM_REFRESH_ALL, NULL); + retro_atomic_store_release_int(&lpl_walk_fail, 1); + ran = refresh_run(items, 1, out, err, true); + retro_atomic_store_release_int(&lpl_walk_fail, 0); + lpl_fail_iter = NULL; + sprintf(what, "refresh %s: a refresh-all whose listing fails", q); + check( ran && chain_saw(0, none_pl, TASK_DBSCAN_FAILED, -1) + && chain_totals.failed == 1 && !chain_totals.ran + && file_contains(err, "out of memory") + && !file_contains(err, "no playlists with saved scan settings"), + what, err); + + /* Cancelled while the folder is walked, then while its second + * playlist loads: nothing after runs. */ + strlcpy(config_get_ptr()->paths.directory_playlist, pl, + sizeof(config_get_ptr()->paths.directory_playlist)); + if (!(items = (task_dbscan_item_t*)calloc(2, sizeof(*items)))) + goto fixture; + refresh_item(&items[0], TASK_DBSCAN_ITEM_REFRESH_ALL, NULL); + chain_item(&items[1], fresh, NULL); + prep_pushes = 0; + prep_cancel_at = 2; + ran = refresh_run(items, 2, out, err, true); + prep_cancel_at = 0; + sprintf(what, "refresh %s: cancelled while walking the folder", q); + sprintf(detail, "%u items, ran %u, done %u%s", chain_seen, + chain_totals.ran, chain_dones, + chain_totals.cancelled ? ", cancelled" : ""); + check( ran && !prep_cancel_task && !chain_seen + && chain_totals.cancelled && !chain_totals.ran + && !chain_totals.failed, what, detail); + if (!(items = (task_dbscan_item_t*)calloc(2, sizeof(*items)))) + goto fixture; + refresh_item(&items[0], TASK_DBSCAN_ITEM_REFRESH_ALL, NULL); + chain_item(&items[1], fresh, NULL); + prep_pushes = 0; + prep_cancel_at = 4; /* the databases, the walk, a_ok, b_gone */ + ran = refresh_run(items, 2, out, err, true); + prep_cancel_at = 0; + sprintf(what, "refresh %s: cancelled while loading a playlist", q); + sprintf(detail, "%u items: %s, ran %u, done %u%s", chain_seen, + chain_names[0], chain_totals.ran, chain_dones, + chain_totals.cancelled ? ", cancelled" : ""); + check( ran && !prep_cancel_task && chain_seen == 1 + && !strcmp(chain_names[0], "a_ok") + && chain_outcomes[0] == TASK_DBSCAN_RAN + && chain_totals.cancelled && chain_totals.ran == 1 + && !chain_totals.skipped && !chain_totals.failed, what, detail); + + /* Without --scan: the outcome, no lines. */ + if (!(items = (task_dbscan_item_t*)calloc(2, sizeof(*items)))) + goto fixture; + refresh_item(&items[0], TASK_DBSCAN_ITEM_REFRESH, "b_gone"); + refresh_item(&items[1], TASK_DBSCAN_ITEM_REFRESH, "a_ok"); + ran = refresh_run(items, 2, out, err, false); + sprintf(what, "refresh %s: nothing prints without --scan", q); + check( ran && chain_saw(0, "b_gone", TASK_DBSCAN_SKIPPED, -1) + && chain_saw(1, "a_ok", TASK_DBSCAN_RAN, 0) + && path_get_size(out) == 0 && path_get_size(err) == 0, what, err); + + manual_content_scan_reset_settings(); + strlcpy(config_get_ptr()->paths.directory_playlist, pl_dir, + sizeof(config_get_ptr()->paths.directory_playlist)); + return; + +fixture: + check(0, "fixture", "could not write the refresh lane"); + strlcpy(config_get_ptr()->paths.directory_playlist, pl_dir, + sizeof(config_get_ptr()->paths.directory_playlist)); +} + int main(int argc, char **argv) { const char *root = (argc > 1) ? argv[1] : "/tmp"; @@ -10444,6 +11036,8 @@ int main(int argc, char **argv) core_lookup_lane(root); chain_lane(root, pl_dir, crc_a, sz_a, false); chain_lane(root, pl_dir, crc_a, sz_a, true); + refresh_lane(root, pl_dir, crc_a, sz_a, false); + refresh_lane(root, pl_dir, crc_a, sz_a, true); { char reader_db[512]; diff --git a/tasks/task_database.c b/tasks/task_database.c index 7243e8bc18c2..47349dc20a51 100644 --- a/tasks/task_database.c +++ b/tasks/task_database.c @@ -611,6 +611,9 @@ typedef struct manual_scan_handle task_dbscan_result_t result; /* What a scan that could not start told the user. */ const char *fail_msg; + /* What the --scan lines call the scan instead of its content + * folder: a refreshed playlist's name. */ + char *cli_name; } manual_scan_handle_t; enum scan_verdict @@ -4320,6 +4323,9 @@ static void free_manual_content_scan_handle(manual_scan_handle_t *manual_scan) free(manual_scan->playlist_directory); manual_scan->playlist_directory = NULL; + free(manual_scan->cli_name); + manual_scan->cli_name = NULL; + #ifdef HAVE_LIBRETRODB if (1) { @@ -5486,6 +5492,13 @@ static bool task_manual_scan_iterate_tick(retro_task_t *task, return finished; } +static const char *manual_scan_cli_name( + const manual_scan_handle_t *manual_scan) +{ + return manual_scan->cli_name + ? manual_scan->cli_name : manual_scan->task_config->content_dir; +} + static void task_manual_content_scan_handler(retro_task_t *task) { uint8_t flg; @@ -5539,7 +5552,7 @@ static void task_manual_content_scan_handler(retro_task_t *task) if (manual_scan->task_config->cli_scan_output) { fprintf(stderr, "%s: %s\n", - manual_scan->task_config->content_dir, + manual_scan_cli_name(manual_scan), manual_scan->fail_msg ? manual_scan->fail_msg : msg_hash_to_str(MSG_MANUAL_CONTENT_SCAN_INVALID_CONFIG)); fflush(stdout); @@ -5630,12 +5643,12 @@ static void task_manual_content_scan_handler(retro_task_t *task) { if (no_database) fprintf(stderr, "%s: %s\n", - manual_scan->task_config->content_dir, msg); + manual_scan_cli_name(manual_scan), msg); else { printf("%s\n", msg); printf("%s: %u added, %u duplicates, %u unmatched, %u errors\n", - manual_scan->task_config->content_dir, + manual_scan_cli_name(manual_scan), manual_scan->result.added, manual_scan->result.duplicates, manual_scan->result.unmatched, @@ -5703,8 +5716,10 @@ static bool task_manual_content_scan_finder(retro_task_t *task, void *user_data) (const char*)user_data, manual_scan->playlist_config.path); } -bool task_push_manual_content_scan( - bool do_menu_refresh, +/* @cli_name: what the --scan lines call the scan, NULL for its + * content folder. */ +static bool task_manual_content_scan_push( + bool do_menu_refresh, const char *cli_name, retro_task_callback_t user_cb, void *user_data) { size_t _len; @@ -5739,6 +5754,9 @@ bool task_push_manual_content_scan( if (!manual_scan->m3u_list) goto error; + if (cli_name && !(manual_scan->cli_name = strdup(cli_name))) + goto error; + manual_scan->playlist_config.capacity = COLLECTION_SIZE; manual_scan->playlist_config.old_format = settings->bools.playlist_use_old_format; manual_scan->playlist_config.compress = settings->bools.playlist_compression; @@ -5848,30 +5866,66 @@ bool task_push_manual_content_scan( return false; } +bool task_push_manual_content_scan( + bool do_menu_refresh, + retro_task_callback_t user_cb, void *user_data) +{ + return task_manual_content_scan_push(do_menu_refresh, NULL, user_cb, + user_data); +} + /* --- The scan chain ----------------------------------------------- */ +enum dbscan_prep_kind +{ + DBSCAN_PREP_DBS = 0, /* the database folder */ + DBSCAN_PREP_LIST, /* the playlist folder */ + DBSCAN_PREP_LOAD /* one playlist */ +}; + +enum dbscan_load_status +{ + DBSCAN_LOAD_OK = 0, + DBSCAN_LOAD_NO_PLAYLIST, + DBSCAN_LOAD_AMBIGUOUS, + DBSCAN_LOAD_NO_SETTINGS, + DBSCAN_LOAD_FOLDER_MISSING +}; + /* A prep task's state: the handler fills it, and the callback takes * what it needs on the main thread once the task finished. */ typedef struct { - char *dir; + playlist_config_t config; /* LOAD: the playlist */ + char *dir; /* DBS, LIST; a --scan-refresh's LOAD */ + char *name; /* LOAD: the name its lines give */ dir_list_iter_t *iter; struct string_list *list; + playlist_parse_t *parse; + playlist_t *playlist; + enum dbscan_prep_kind kind; + enum dbscan_load_status load; + unsigned matches; /* LOAD: files in another case, if ambiguous */ bool include_hidden; bool begun; + bool failed; /* LIST: out of memory */ } dbscan_prep_t; typedef struct { task_dbscan_item_t *items; struct string_list *rdb_list; + struct string_list *playlists; /* the running refresh-all's */ char *name; /* the running scan's */ task_dbscan_item_cb_t item_cb; task_dbscan_chain_cb_t done_cb; void *user_data; size_t count; size_t next; + size_t all_next; task_dbscan_chain_result_t totals; + bool all_running; + bool all_counted; /* a playlist of the refresh-all had settings */ bool cli_output; } dbscan_chain_t; @@ -5900,6 +5954,8 @@ static void dbscan_chain_finish(dbscan_chain_t *chain) dbscan_chain_free_items(chain->items, chain->count); if (chain->rdb_list) string_list_free(chain->rdb_list); + if (chain->playlists) + string_list_free(chain->playlists); free(chain->name); free(chain); dbscan_chain_running = NULL; @@ -5926,17 +5982,14 @@ static void dbscan_chain_fold(dbscan_chain_t *chain, const char *name, chain->item_cb(name, outcome, res, chain->user_data); } -/* @in with '~' expanded, made absolute if relative; NULL, with the - * reason on stderr, if nothing is there. Free the result. */ -static char *dbscan_chain_resolve(const char *in) +/* @in with '~' expanded, made absolute if relative; NULL on + * allocation failure. Free the result. */ +static char *dbscan_chain_expand(const char *in) { char *s = (char*)malloc(PATH_MAX_LENGTH); if (!s) - { - fprintf(stderr, "out of memory\n"); return NULL; - } fill_pathname_expand_special(s, in, PATH_MAX_LENGTH); /* "" would resolve to the working directory. Consoles resolve * nothing, so a relative path stays as it is there. */ @@ -5945,6 +5998,20 @@ static char *dbscan_chain_resolve(const char *in) && !path_resolve_realpath(s, PATH_MAX_LENGTH, false)) *s = '\0'; #endif + return s; +} + +/* @in expanded; NULL, with the reason on stderr, if nothing is + * there. Free the result. */ +static char *dbscan_chain_resolve(const char *in) +{ + char *s = dbscan_chain_expand(in); + + if (!s) + { + fprintf(stderr, "out of memory\n"); + return NULL; + } if (!*s || !path_is_valid(s)) { fprintf(stderr, "%s: not found\n", in); @@ -5989,6 +6056,25 @@ static void dbscan_chain_override_error(const dbscan_chain_t *chain, } } +/* A refresh that does not run: logged, and its line with --scan. + * @detail, when not NULL, follows in brackets. */ +static void dbscan_chain_refresh_error(const dbscan_chain_t *chain, + const char *name, const char *reason, const char *detail) +{ + if (detail) + { + RARCH_ERR("[Playlist Refresh] %s: %s (%s).\n", name, reason, detail); + if (chain->cli_output) + fprintf(stderr, "%s: %s (%s)\n", name, reason, detail); + } + else + { + RARCH_ERR("[Playlist Refresh] %s: %s.\n", name, reason); + if (chain->cli_output) + fprintf(stderr, "%s: %s\n", name, reason); + } +} + static void dbscan_chain_next(dbscan_chain_t *chain); static void dbscan_chain_scan_cb(retro_task_t *task, void *task_data, @@ -6077,29 +6163,103 @@ static bool dbscan_chain_push_path(dbscan_chain_t *chain, return false; } -/* Runs items until one pushes a task; after the last, finishes. */ -static void dbscan_chain_next(dbscan_chain_t *chain) +/* Sets the loaded playlist's saved settings, @item's overrides over + * them, and pushes its scan; otherwise folds it as failed. */ +static bool dbscan_chain_push_refreshed(dbscan_chain_t *chain, + const task_dbscan_item_t *item, dbscan_prep_t *prep) { - while (chain->next < chain->count) + enum manual_content_scan_override_status st; + unsigned count = 0; + const char *reason = NULL; + const char *detail = NULL; + playlist_t *pl = prep->playlist; + + manual_content_scan_reset_settings(); + switch (manual_content_scan_set_menu_from_playlist_list(pl, + chain->rdb_list)) + { + case MANUAL_CONTENT_SCAN_PLAYLIST_REFRESH_OK: + break; + case MANUAL_CONTENT_SCAN_PLAYLIST_REFRESH_INVALID_CONTENT_DIR: + reason = "invalid content folder"; + detail = playlist_get_scan_content_dir(pl); + break; + case MANUAL_CONTENT_SCAN_PLAYLIST_REFRESH_INVALID_SYSTEM_NAME: + reason = "invalid playlist name"; + break; + case MANUAL_CONTENT_SCAN_PLAYLIST_REFRESH_INVALID_CORE: + reason = "saved core not found"; + detail = playlist_get_default_core_name(pl); + break; + case MANUAL_CONTENT_SCAN_PLAYLIST_REFRESH_INVALID_DAT_FILE: + reason = "not a DAT file"; + detail = playlist_get_scan_dat_file_path(pl); + break; + case MANUAL_CONTENT_SCAN_PLAYLIST_REFRESH_DAT_FILE_TOO_LARGE: + reason = "DAT file too large"; + detail = playlist_get_scan_dat_file_path(pl); + break; + case MANUAL_CONTENT_SCAN_PLAYLIST_REFRESH_MISSING_CONFIG: + default: + reason = "no saved scan settings"; + break; + } + if (reason) + dbscan_chain_refresh_error(chain, prep->name, reason, detail); + playlist_free(pl); + prep->playlist = NULL; + if (!reason) { - const task_dbscan_item_t *item = &chain->items[chain->next++]; + if ((st = manual_content_scan_apply_overrides(item->overrides, + chain->rdb_list, &count)) != MANUAL_CONTENT_SCAN_OVR_OK) + dbscan_chain_override_error(chain, prep->name, item->overrides, + st, count); + else + { + dbscan_chain_single_warn(chain, prep->name, + item->overrides ? item->overrides->flags : 0); + if (task_manual_content_scan_push(false, prep->name, + dbscan_chain_scan_cb, chain)) + { + chain->name = prep->name; + prep->name = NULL; + return true; + } + dbscan_chain_refresh_error(chain, prep->name, + "the scan could not start", NULL); + } + } + dbscan_chain_fold(chain, prep->name, TASK_DBSCAN_FAILED, NULL); + return false; +} - switch (item->kind) +/* Drops what Manage Playlists does not list: the histories and the + * favorites. */ +static void dbscan_prep_filter(struct string_list *list) +{ + size_t i; + size_t n = 0; + + for (i = 0; i < list->size; i++) + { + const char *path = list->elems[i].data; + + if ( string_ends_with_size(path, "_history.lpl", strlen(path), + STRLEN_CONST("_history.lpl")) + || string_is_equal(path_basename_nocompression(path), + FILE_PATH_CONTENT_FAVORITES)) { - case TASK_DBSCAN_ITEM_PATH: - if (dbscan_chain_push_path(chain, item)) - return; - break; - default: - dbscan_chain_fold(chain, item->arg, TASK_DBSCAN_FAILED, NULL); - break; + free(list->elems[i].data); + free(list->elems[i].userdata); } + else + list->elems[n++] = list->elems[i]; } - dbscan_chain_finish(chain); + list->size = n; } -/* Lists the database folder: to the end on a threaded queue's worker, - * else under the shared window. */ +/* Lists the database or playlist folder: to the end on a threaded + * queue's worker, else under the shared window. */ static void dbscan_prep_handler(retro_task_t *task) { dbscan_prep_t *prep = (dbscan_prep_t*)task->state; @@ -6114,7 +6274,8 @@ static void dbscan_prep_handler(retro_task_t *task) { prep->begun = true; if (prep->dir && (prep->list = string_list_new())) - prep->iter = dir_list_iter_new(prep->dir, "rdb", false, + prep->iter = dir_list_iter_new(prep->dir, + prep->kind == DBSCAN_PREP_LIST ? "lpl" : "rdb", false, prep->include_hidden, false, false, prep->list); } if (prep->iter) @@ -6132,12 +6293,19 @@ static void dbscan_prep_handler(retro_task_t *task) return; } } + prep->failed = r < 0 || (prep->dir && !prep->list); /* No listing, as dir_list_new() gives none. */ if ((!prep->iter || r < 0) && prep->list) { string_list_free(prep->list); prep->list = NULL; } + /* In the order Manage Playlists shows. */ + if (prep->kind == DBSCAN_PREP_LIST && prep->list) + { + dbscan_prep_filter(prep->list); + dir_list_sort(prep->list, true); + } task_set_flags(task, RETRO_TASK_FLG_FINISHED, true); } @@ -6147,15 +6315,222 @@ static void dbscan_prep_cb(retro_task_t *task, void *task_data, dbscan_chain_t *chain = (dbscan_chain_t*)user_data; dbscan_prep_t *prep = (dbscan_prep_t*)task->state; - chain->rdb_list = prep->list; - prep->list = NULL; + if (prep->kind == DBSCAN_PREP_LIST) + { + chain->playlists = prep->list; + chain->all_next = 0; + chain->all_running = !prep->failed; + chain->all_counted = false; + } + else + chain->rdb_list = prep->list; + prep->list = NULL; if (task_get_flags(task) & RETRO_TASK_FLG_CANCELLED) { chain->totals.cancelled = true; dbscan_chain_finish(chain); + return; } + if (prep->kind == DBSCAN_PREP_LIST && prep->failed) + { + if (chain->cli_output) + fprintf(stderr, "out of memory\n"); + dbscan_chain_fold(chain, prep->dir, TASK_DBSCAN_FAILED, NULL); + } + dbscan_chain_next(chain); +} + +/* Names ignore case, as the file system may: the name and the path + * come from the folder's listing, the exact name first, since a save + * onto the path as typed can rename the file. Two in other cases are + * ambiguous. False while the listing needs another step. */ +static bool dbscan_load_name_step(dbscan_prep_t *prep) +{ + size_t i; + int r; + unsigned matches = 0; + const char *want = path_basename(prep->config.path); + const char *match = NULL; + + if (task_queue_is_threaded()) + r = dir_list_iter_step(prep->iter, NULL, NULL); else - dbscan_chain_next(chain); + { + nbio_budget_t b; + task_nbio_slice_open(&b); + r = dir_list_iter_step(prep->iter, + manual_scan_walk_within_budget, &b); + task_nbio_slice_close(&b); + if (!r) + return false; + } + for (i = 0; r > 0 && i < prep->list->size; i++) + { + const char *path = prep->list->elems[i].data; + + if (string_is_equal(path_basename(path), want)) + { + match = path; + matches = 1; + break; + } + if ( string_is_equal_case_insensitive(path_basename(path), want) + && !matches++) + match = path; + } + if (matches > 1) + { + prep->matches = matches; + prep->load = DBSCAN_LOAD_AMBIGUOUS; + } + else if (match) + { + fill_pathname(prep->name, path_basename(match), "", NAME_MAX_LENGTH); + playlist_config_set_path(&prep->config, match); + } + dir_list_iter_free(prep->iter); + prep->iter = NULL; + return true; +} + +/* Loads one playlist and tells whether it can be refreshed: to the end + * on a threaded queue's worker, else under the shared window. */ +static void dbscan_load_handler(retro_task_t *task) +{ + dbscan_prep_t *prep = (dbscan_prep_t*)task->state; + + if (task_get_flags(task) & RETRO_TASK_FLG_CANCELLED) + { + task_set_flags(task, RETRO_TASK_FLG_FINISHED, true); + return; + } + if (!prep->begun) + { + prep->begun = true; + /* The parse waits for the listed path. */ + if (prep->dir && (prep->list = string_list_new())) + prep->iter = dir_list_iter_new(prep->dir, "lpl", false, true, + false, false, prep->list); + if (prep->iter) + return; + if (!path_is_valid(prep->config.path)) + prep->load = DBSCAN_LOAD_NO_PLAYLIST; + else + prep->parse = playlist_parse_begin(&prep->config); + } + /* One window per call on the main thread. */ + if (prep->iter) + { + if (!dbscan_load_name_step(prep)) + return; + if (prep->load == DBSCAN_LOAD_OK) + { + if (!path_is_valid(prep->config.path)) + prep->load = DBSCAN_LOAD_NO_PLAYLIST; + else + prep->parse = playlist_parse_begin(&prep->config); + } + if (!task_queue_is_threaded()) + return; + } + if (prep->parse) + { + int r; + + if (task_queue_is_threaded()) + r = playlist_parse_step(prep->parse, NULL, NULL); + else + { + nbio_budget_t b; + task_nbio_slice_open(&b); + r = playlist_parse_step(prep->parse, + manual_scan_walk_within_budget, &b); + task_nbio_slice_close(&b); + if (!r) + return; + } + if (r < 0) + playlist_parse_abort(prep->parse); + else + prep->playlist = playlist_parse_end(prep->parse); + prep->parse = NULL; + } + if (prep->load == DBSCAN_LOAD_OK) + { + char *dir = NULL; + + /* A playlist that does not parse has no settings to read. */ + if (!playlist_scan_refresh_enabled(prep->playlist)) + prep->load = DBSCAN_LOAD_NO_SETTINGS; + else if ((dir = (char*)malloc(PATH_MAX_LENGTH))) + { + fill_pathname_expand_special(dir, + playlist_get_scan_content_dir(prep->playlist), + PATH_MAX_LENGTH); + /* A single file's scan saves the file. */ + if (!path_is_valid(dir)) + prep->load = DBSCAN_LOAD_FOLDER_MISSING; + free(dir); + } + } + task_set_flags(task, RETRO_TASK_FLG_FINISHED, true); +} + +static void dbscan_load_cb(retro_task_t *task, void *task_data, + void *user_data, const char *err) +{ + dbscan_chain_t *chain = (dbscan_chain_t*)user_data; + dbscan_prep_t *prep = (dbscan_prep_t*)task->state; + const task_dbscan_item_t *item = &chain->items[chain->next - 1]; + /* A --scan-refresh's own failures name it as typed. */ + const char *arg = item->kind == TASK_DBSCAN_ITEM_REFRESH + ? item->arg : prep->name; + + if (task_get_flags(task) & RETRO_TASK_FLG_CANCELLED) + { + chain->totals.cancelled = true; + dbscan_chain_finish(chain); + return; + } + if (prep->load != DBSCAN_LOAD_NO_SETTINGS) + chain->all_counted = true; + switch (prep->load) + { + case DBSCAN_LOAD_NO_PLAYLIST: + dbscan_chain_refresh_error(chain, arg, "no such playlist", NULL); + dbscan_chain_fold(chain, arg, TASK_DBSCAN_FAILED, NULL); + break; + case DBSCAN_LOAD_AMBIGUOUS: + { + char reason[80]; + snprintf(reason, sizeof(reason), + "%u playlists with this name in other cases, " + "give the exact name", + prep->matches); + dbscan_chain_refresh_error(chain, arg, reason, NULL); + dbscan_chain_fold(chain, arg, TASK_DBSCAN_FAILED, NULL); + } + break; + case DBSCAN_LOAD_NO_SETTINGS: + /* A refresh-all neither lists nor counts it. */ + if (item->kind == TASK_DBSCAN_ITEM_REFRESH_ALL) + break; + dbscan_chain_refresh_error(chain, arg, "no saved scan settings", + NULL); + dbscan_chain_fold(chain, arg, TASK_DBSCAN_FAILED, NULL); + break; + case DBSCAN_LOAD_FOLDER_MISSING: + dbscan_chain_refresh_error(chain, prep->name, + "skipped: content folder not found", + playlist_get_scan_content_dir(prep->playlist)); + dbscan_chain_fold(chain, prep->name, TASK_DBSCAN_SKIPPED, NULL); + break; + case DBSCAN_LOAD_OK: + if (dbscan_chain_push_refreshed(chain, item, prep)) + return; + break; + } + dbscan_chain_next(chain); } static void dbscan_prep_free(retro_task_t *task) @@ -6167,10 +6542,240 @@ static void dbscan_prep_free(retro_task_t *task) dir_list_iter_free(prep->iter); if (prep->list) string_list_free(prep->list); + playlist_parse_abort(prep->parse); + playlist_free(prep->playlist); + free(prep->name); free(prep->dir); free(prep); } +/* Pushes a prep task for @prep; false, @prep freed, if it cannot. */ +static bool dbscan_chain_push_prep(dbscan_chain_t *chain, + dbscan_prep_t *prep, retro_task_handler_t handler, + retro_task_callback_t cb) +{ + retro_task_t *task = prep ? task_init() : NULL; + + if (!task) + { + if (prep) + { + free(prep->name); + free(prep->dir); + free(prep); + } + return false; + } + /* Silent: no title, no notifications. */ + task->handler = handler; + task->state = prep; + task->user_data = chain; + task->callback = cb; + task->cleanup = dbscan_prep_free; + task_queue_push(task); + return true; +} + +/* Loads the playlist at @path off the main thread; otherwise folds it + * as failed. */ +static bool dbscan_chain_push_load(dbscan_chain_t *chain, + const char *path) +{ + char name[NAME_MAX_LENGTH]; + settings_t *settings = config_get_ptr(); + dbscan_prep_t *prep = (dbscan_prep_t*)calloc(1, sizeof(*prep)); + + if (prep && !(prep->name = (char*)malloc(NAME_MAX_LENGTH))) + { + free(prep); + prep = NULL; + } + if (prep) + { + fill_pathname(prep->name, path_basename(path), "", NAME_MAX_LENGTH); + /* A refresh-all's names come from its listing already. */ + if ( chain->items[chain->next - 1].kind == TASK_DBSCAN_ITEM_REFRESH + && (prep->dir = (char*)malloc(PATH_MAX_LENGTH))) + { + fill_pathname_basedir(prep->dir, path, PATH_MAX_LENGTH); +#ifdef _WIN32 + pathname_conform_slashes_to_os(prep->dir); +#endif + } + /* As Manage Playlists opens it. */ + playlist_config_set_path(&prep->config, path); + prep->config.capacity = COLLECTION_SIZE; + prep->config.old_format = settings->bools.playlist_use_old_format; + prep->config.compress = settings->bools.playlist_compression; + prep->config.fuzzy_archive_match = settings->bools.playlist_fuzzy_archive_match; + playlist_config_set_base_content_directory(&prep->config, + settings->bools.playlist_portable_paths + ? settings->paths.directory_menu_content : NULL); + prep->kind = DBSCAN_PREP_LOAD; + } + if (dbscan_chain_push_prep(chain, prep, dbscan_load_handler, + dbscan_load_cb)) + return true; + fill_pathname(name, path_basename(path), "", sizeof(name)); + dbscan_chain_refresh_error(chain, name, "the scan could not start", NULL); + dbscan_chain_fold(chain, name, TASK_DBSCAN_FAILED, NULL); + return false; +} + +/* The playlist @arg names, or NULL, with its line, if it names none + * in the playlist folder. Free the result. */ +static char *dbscan_chain_refresh_path(const dbscan_chain_t *chain, + const char *arg) +{ + const char *pl_dir = config_get_ptr()->paths.directory_playlist; + size_t _len = strlen(arg); + char *path = NULL; + char *dir = NULL; + char *want = NULL; + + /* A playlist name cannot hold a slash; a file name is one too. */ + if (!find_last_slash(arg)) + { + if ((path = (char*)malloc(PATH_MAX_LENGTH))) + { + fill_pathname_join_special(path, pl_dir, arg, PATH_MAX_LENGTH); + if (!( _len >= STRLEN_CONST(".lpl") + && string_is_equal_case_insensitive( + arg + _len - STRLEN_CONST(".lpl"), ".lpl"))) + strlcat(path, ".lpl", PATH_MAX_LENGTH); + } + else if (chain->cli_output) + fprintf(stderr, "out of memory\n"); + return path; + } + path = dbscan_chain_expand(arg); + dir = dbscan_chain_expand(pl_dir); + want = (char*)malloc(PATH_MAX_LENGTH); + if (path && dir && want) + { + /* "//", "/./" and ".." would fail the comparison; @dir stays + * as configured for the line. */ + path_resolve_realpath(path, PATH_MAX_LENGTH, false); + /* The scan writes into the playlist folder, whatever the path. */ + fill_pathname_join_special(want, dir, path_basename(path), + PATH_MAX_LENGTH); + path_resolve_realpath(want, PATH_MAX_LENGTH, false); +#ifdef _WIN32 + pathname_conform_slashes_to_os(path); + pathname_conform_slashes_to_os(want); + if (!string_is_equal_case_insensitive(path, want)) +#else + if (!string_is_equal(path, want)) +#endif + { + dbscan_chain_refresh_error(chain, arg, + "not in the playlist folder", dir); + free(path); + path = NULL; + } + } + else + { + if (chain->cli_output) + fprintf(stderr, "out of memory\n"); + free(path); + path = NULL; + } + free(dir); + free(want); + return path; +} + +/* True once @item's playlist is loading; otherwise it is folded as + * failed. */ +static bool dbscan_chain_push_refresh(dbscan_chain_t *chain, + const task_dbscan_item_t *item) +{ + bool pushed = false; + char *path = dbscan_chain_refresh_path(chain, item->arg); + + if (path) + pushed = dbscan_chain_push_load(chain, path); + else + dbscan_chain_fold(chain, item->arg, TASK_DBSCAN_FAILED, NULL); + free(path); + return pushed; +} + +/* Lists the playlist folder off the main thread when a refresh-all is + * reached, so it takes what earlier items wrote. */ +static bool dbscan_chain_push_list(dbscan_chain_t *chain) +{ + settings_t *settings = config_get_ptr(); + dbscan_prep_t *prep = (dbscan_prep_t*)calloc(1, sizeof(*prep)); + + if (prep) + { + prep->kind = DBSCAN_PREP_LIST; + prep->include_hidden = settings->bools.show_hidden_files; + if ( *settings->paths.directory_playlist + && !(prep->dir = strdup(settings->paths.directory_playlist))) + { + free(prep); + prep = NULL; + } + } + return dbscan_chain_push_prep(chain, prep, dbscan_prep_handler, + dbscan_prep_cb); +} + +/* Runs items, and a refresh-all's playlists, until one pushes a task; + * after the last, finishes. */ +static void dbscan_chain_next(dbscan_chain_t *chain) +{ + for (;;) + { + const task_dbscan_item_t *item; + + if (chain->all_running) + { + if ( chain->playlists + && chain->all_next < chain->playlists->size) + { + if (dbscan_chain_push_load(chain, + chain->playlists->elems[chain->all_next++].data)) + return; + continue; + } + if (!chain->all_counted && chain->cli_output) + fprintf(stderr, "no playlists with saved scan settings\n"); + if (chain->playlists) + string_list_free(chain->playlists); + chain->playlists = NULL; + chain->all_running = false; + } + if (chain->next >= chain->count) + break; + item = &chain->items[chain->next++]; + switch (item->kind) + { + case TASK_DBSCAN_ITEM_PATH: + if (dbscan_chain_push_path(chain, item)) + return; + break; + case TASK_DBSCAN_ITEM_REFRESH: + if (dbscan_chain_push_refresh(chain, item)) + return; + break; + case TASK_DBSCAN_ITEM_REFRESH_ALL: + if (dbscan_chain_push_list(chain)) + return; + if (chain->cli_output) + fprintf(stderr, "out of memory\n"); + dbscan_chain_fold(chain, + config_get_ptr()->paths.directory_playlist, + TASK_DBSCAN_FAILED, NULL); + break; + } + } + dbscan_chain_finish(chain); +} + /* Any unfinished scan, whatever its playlist. */ static bool dbscan_chain_scan_finder(retro_task_t *task, void *user_data) { @@ -6183,7 +6788,6 @@ bool task_push_dbscan_chain(task_dbscan_item_t *items, size_t count, void *user_data) { task_finder_data_t find_data; - retro_task_t *task = NULL; dbscan_chain_t *chain = NULL; dbscan_prep_t *prep = NULL; settings_t *settings = config_get_ptr(); @@ -6193,14 +6797,14 @@ bool task_push_dbscan_chain(task_dbscan_item_t *items, size_t count, if (dbscan_chain_running || task_queue_find(&find_data)) goto error; if ( !(chain = (dbscan_chain_t*)calloc(1, sizeof(*chain))) - || !(prep = (dbscan_prep_t*)calloc(1, sizeof(*prep))) - || !(task = task_init())) + || !(prep = (dbscan_prep_t*)calloc(1, sizeof(*prep)))) goto error; #ifdef HAVE_LIBRETRODB if ( *settings->paths.path_content_database && !(prep->dir = strdup(settings->paths.path_content_database))) goto error; #endif + prep->kind = DBSCAN_PREP_DBS; prep->include_hidden = settings->bools.show_hidden_files; chain->items = items; @@ -6211,18 +6815,16 @@ bool task_push_dbscan_chain(task_dbscan_item_t *items, size_t count, chain->cli_output = retroarch_override_setting_is_set( RARCH_OVERRIDE_SETTING_DATABASE_SCAN, NULL); - /* Silent: no title, no notifications. */ - task->handler = dbscan_prep_handler; - task->state = prep; - task->user_data = chain; - task->callback = dbscan_prep_cb; - task->cleanup = dbscan_prep_free; dbscan_chain_running = chain; - task_queue_push(task); - return true; + if (dbscan_chain_push_prep(chain, prep, dbscan_prep_handler, + dbscan_prep_cb)) + return true; + dbscan_chain_running = NULL; + prep = NULL; error: - free(task); + if (prep) + free(prep->dir); free(prep); free(chain); dbscan_chain_free_items(items, count); From 0c14980c2452780d4af420431e08cd72e3053a5a Mon Sep 17 00:00:00 2001 From: davey Date: Sun, 1 Nov 2026 03:05:48 -0800 Subject: [PATCH 15/21] retroarch: --scan-refresh and --scan-refresh-all --scan-refresh=NAME|PATH rescans a playlist with its saved scan settings, as the menu's Refresh Playlist does, and --scan-refresh-all rescans every playlist that has them. Both are items of the same run as --scan: the scan options after one are laid over the saved settings and written back, and the exit status follows the same rule. --scan-playlist after a refresh is refused, so a refresh always updates its own playlist, and an empty --scan-refresh is refused as the other empty values are. Like --scan, the refresh options cannot be combined with --cloudsync or --command; the error names whichever of the three comes first on the command line. --- retroarch.c | 76 ++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 63 insertions(+), 13 deletions(-) diff --git a/retroarch.c b/retroarch.c index 9fa940ba1012..49dc2e1aaf76 100644 --- a/retroarch.c +++ b/retroarch.c @@ -391,6 +391,8 @@ enum RA_OPT_MAX_FRAMES_SCREENSHOT_PATH, RA_OPT_SET_SHADER, RA_OPT_DATABASE_SCAN, + RA_OPT_SCAN_REFRESH, + RA_OPT_SCAN_REFRESH_ALL, RA_OPT_SCAN_PLAYLIST, RA_OPT_SCAN_DATABASE, RA_OPT_SCAN_MATCH, @@ -7949,9 +7951,16 @@ static void retroarch_print_help(const char *arg0) " " " The options from --scan-playlist on apply\n" " " - " to the --scan before them. Without any,\n" + " to the --scan, --scan-refresh or\n" " " - " the scan is Fully Automatic.\n"); + " --scan-refresh-all before them. Without\n" + " " + " any, the scan is Fully Automatic.\n"); + strlcpy_append(buf, sizeof(buf), &_len, + " --scan-refresh=NAME|PATH " + "Refresh a playlist by its saved scan settings.\n" + " --scan-refresh-all " + "Refresh each playlist that has saved settings.\n"); strlcpy_append(buf, sizeof(buf), &_len, " --scan-playlist=NAME " "Playlist to add to.\n" @@ -8376,9 +8385,17 @@ static void retroarch_cli_scan_option(task_dbscan_item_t *items, switch (c) { case RA_OPT_SCAN_PLAYLIST: - field = &ov->playlist; - /* .lpl.tmp fits a file name */ - max_len = NAME_MAX_LENGTH - 9; + /* A refresh writes its own playlist. */ + if (items[count - 1].kind == TASK_DBSCAN_ITEM_REFRESH) + err = "cannot follow --scan-refresh"; + else if (items[count - 1].kind == TASK_DBSCAN_ITEM_REFRESH_ALL) + err = "cannot follow --scan-refresh-all"; + else + { + field = &ov->playlist; + /* .lpl.tmp fits a file name */ + max_len = NAME_MAX_LENGTH - 9; + } break; case RA_OPT_SCAN_DATABASE: if (string_is_equal(copy, "first")) @@ -8674,6 +8691,8 @@ static bool retroarch_parse_input_and_config( { "entryslot", 1, NULL, 'e' }, #ifdef HAVE_LIBRETRODB { "scan", 1, NULL, RA_OPT_DATABASE_SCAN }, + { "scan-refresh", 1, NULL, RA_OPT_SCAN_REFRESH }, + { "scan-refresh-all", 0, NULL, RA_OPT_SCAN_REFRESH_ALL }, { "scan-playlist", 1, NULL, RA_OPT_SCAN_PLAYLIST }, { "scan-database", 1, NULL, RA_OPT_SCAN_DATABASE }, { "scan-match", 1, NULL, RA_OPT_SCAN_MATCH }, @@ -8879,15 +8898,37 @@ static bool retroarch_parse_input_and_config( explicit_menu = true; break; case RA_OPT_DATABASE_SCAN: + case RA_OPT_SCAN_REFRESH: + case RA_OPT_SCAN_REFRESH_ALL: #ifdef HAVE_LIBRETRODB - cli_scan = true; - verbosity_enable(); - retroarch_override_setting_set(RARCH_OVERRIDE_SETTING_DATABASE_SCAN, NULL); - if (!retroarch_cli_scan_add(&scan_items, &scan_count, - &scan_cap, TASK_DBSCAN_ITEM_PATH, optarg)) { - fprintf(stderr, "out of memory\n"); - exit(1); + enum task_dbscan_item_kind kind = TASK_DBSCAN_ITEM_PATH; + const char *arg = optarg; + + if (c == RA_OPT_SCAN_REFRESH) + { + if (!*arg) + { + frontend_driver_attach_console(); + fprintf(stderr, "--scan-refresh needs a value\n"); + exit(1); + } + kind = TASK_DBSCAN_ITEM_REFRESH; + } + else if (c == RA_OPT_SCAN_REFRESH_ALL) + { + kind = TASK_DBSCAN_ITEM_REFRESH_ALL; + arg = NULL; + } + cli_scan = true; + verbosity_enable(); + retroarch_override_setting_set(RARCH_OVERRIDE_SETTING_DATABASE_SCAN, NULL); + if (!retroarch_cli_scan_add(&scan_items, &scan_count, + &scan_cap, kind, arg)) + { + fprintf(stderr, "out of memory\n"); + exit(1); + } } #endif break; @@ -8947,7 +8988,14 @@ static bool retroarch_parse_input_and_config( /* Each of them is the whole run. */ if (cli_scan && scan_conflict) { - fprintf(stderr, "--scan cannot be combined with %s\n", scan_conflict); + const char *first = "--scan"; + + if (scan_items[0].kind == TASK_DBSCAN_ITEM_REFRESH) + first = "--scan-refresh"; + else if (scan_items[0].kind == TASK_DBSCAN_ITEM_REFRESH_ALL) + first = "--scan-refresh-all"; + fprintf(stderr, "%s cannot be combined with %s\n", first, + scan_conflict); exit(1); } #endif @@ -9372,6 +9420,8 @@ static bool retroarch_parse_input_and_config( } break; case RA_OPT_DATABASE_SCAN: + case RA_OPT_SCAN_REFRESH: + case RA_OPT_SCAN_REFRESH_ALL: case RA_OPT_SCAN_PLAYLIST: case RA_OPT_SCAN_DATABASE: case RA_OPT_SCAN_MATCH: From 011515e1a77a7fadc2da07d702454ac0344a57a2 Mon Sep 17 00:00:00 2001 From: davey Date: Sun, 1 Nov 2026 03:36:02 -0800 Subject: [PATCH 16/21] menu: Refresh All Playlists Manage Playlists gets Refresh All Playlists as its first entry when it lists any playlist, so an empty list still says there are no entries. It rescans every playlist that has saved scan settings, with those settings, through the same chain as --scan-refresh-all, and ends with one notification saying how many playlists were refreshed, skipped (content folder gone) and failed. While a content scan runs it is refused with a new message; a refused scan had none. Other languages come through Crowdin. While a scan chain runs (Refresh All, or a command-line scan with --menu), Start Scan, a playlist's Refresh, Scan Directory and Scan File are refused with the same message, as they would run with the settings the chain last loaded. task_push_dbscan() returns false then too, which covers the companion and the iOS caller. --- intl/msg_hash_lbl.h | 4 + intl/msg_hash_us.h | 16 +++ menu/cbs/menu_cbs_ok.c | 49 +++++++ menu/cbs/menu_cbs_scan.c | 19 +++ menu/cbs/menu_cbs_sublabel.c | 1 + menu/drivers/materialui.c | 1 + menu/drivers/ozone.c | 1 + menu/drivers/xmb.c | 1 + menu/menu_displaylist.c | 11 ++ msg_hash.h | 4 + msg_hash_lbl_str.h | 2 + samples/tasks/database/database_scan_test.c | 142 +++++++++++++++++++- tasks/task_database.c | 22 ++- tasks/tasks_internal.h | 9 +- 14 files changed, 276 insertions(+), 6 deletions(-) diff --git a/intl/msg_hash_lbl.h b/intl/msg_hash_lbl.h index c6dc28b3a804..bd8f511833f3 100644 --- a/intl/msg_hash_lbl.h +++ b/intl/msg_hash_lbl.h @@ -1955,6 +1955,10 @@ MSG_HASH( MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_PLAYLIST, "playlist_manager_refresh_playlist" ) +MSG_HASH( + MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_ALL_PLAYLISTS, + "playlist_manager_refresh_all_playlists" + ) MSG_HASH( MENU_ENUM_LABEL_PLAYLIST_SETTINGS_BEGIN, "playlist_settings_begin" diff --git a/intl/msg_hash_us.h b/intl/msg_hash_us.h index 4a66492a66c5..57b61aadaf52 100644 --- a/intl/msg_hash_us.h +++ b/intl/msg_hash_us.h @@ -23216,6 +23216,14 @@ MSG_HASH( MENU_ENUM_SUBLABEL_PLAYLIST_MANAGER_REFRESH_PLAYLIST, "Add new content and remove invalid entries by repeating the content scan operation last used to create or edit the playlist." ) +MSG_HASH( + MENU_ENUM_LABEL_VALUE_PLAYLIST_MANAGER_REFRESH_ALL_PLAYLISTS, + "Refresh All Playlists" + ) +MSG_HASH( + MENU_ENUM_SUBLABEL_PLAYLIST_MANAGER_REFRESH_ALL_PLAYLISTS, + "Rescan every playlist that has saved scan settings, with those settings." + ) MSG_HASH( MENU_ENUM_LABEL_VALUE_DELETE_PLAYLIST, "Delete Playlist" @@ -32548,6 +32556,10 @@ MSG_HASH( MSG_PLAYLIST_MANAGER_REFRESH_DAT_FILE_TOO_LARGE, "Refresh failed - arcade DAT file too large (insufficient memory): " ) +MSG_HASH( + MSG_PLAYLIST_MANAGER_REFRESH_ALL_DONE, + "Playlists refreshed: %u, skipped: %u, failed: %u" + ) MSG_HASH( MSG_ADDED_TO_FAVORITES, "Added to favorites" @@ -34344,6 +34356,10 @@ MSG_HASH( MSG_MANUAL_CONTENT_SCAN_END, "Scan complete: " ) +MSG_HASH( + MSG_MANUAL_CONTENT_SCAN_ALREADY_RUNNING, + "A content scan is already running." + ) MSG_HASH( MSG_CORE_BACKUP_SCANNING_CORE, "Scanning core: " diff --git a/menu/cbs/menu_cbs_ok.c b/menu/cbs/menu_cbs_ok.c index c391cbbf49bf..f09a0b8eb6c9 100644 --- a/menu/cbs/menu_cbs_ok.c +++ b/menu/cbs/menu_cbs_ok.c @@ -9123,6 +9123,14 @@ static int action_ok_manual_content_scan_start(const char *path, { playlist_config_t playlist_config; settings_t *settings = config_get_ptr(); + if (task_dbscan_chain_is_running()) + { + const char *_msg = msg_hash_to_str( + MSG_MANUAL_CONTENT_SCAN_ALREADY_RUNNING); + runloop_msg_queue_push(_msg, strlen(_msg), 1, 150, true, NULL, + MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_ERROR); + return 0; + } /* Note: playlist_config.path will set by the * task itself */ playlist_config.capacity = COLLECTION_SIZE; @@ -9599,6 +9607,14 @@ static int action_ok_playlist_refresh(const char *path, playlist_config = playlist_get_config(playlist); if (!playlist_config || !*playlist_config->path) return -1; + if (task_dbscan_chain_is_running()) + { + const char *_msg = msg_hash_to_str( + MSG_MANUAL_CONTENT_SCAN_ALREADY_RUNNING); + runloop_msg_queue_push(_msg, strlen(_msg), 1, 150, true, NULL, + MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_ERROR); + return 0; + } stat = manual_content_scan_set_menu_from_playlist(playlist, settings->paths.path_content_database, settings->bools.show_hidden_files); @@ -9688,6 +9704,38 @@ static int action_ok_playlist_refresh(const char *path, return 0; } +static void action_ok_playlist_refresh_all_cb( + const task_dbscan_chain_result_t *res, void *user_data) +{ + char msg[128]; + size_t _len = snprintf(msg, sizeof(msg), + msg_hash_to_str(MSG_PLAYLIST_MANAGER_REFRESH_ALL_DONE), + res->ran, res->skipped, res->failed); + runloop_msg_queue_push(msg, _len, 1, 180, true, NULL, + MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_INFO); +} + +static int action_ok_playlist_refresh_all(const char *path, + const char *label, unsigned type, size_t idx, size_t entry_idx) +{ + task_dbscan_item_t *item = NULL; + if (!task_dbscan_chain_can_start()) + { + const char *_msg = msg_hash_to_str( + MSG_MANUAL_CONTENT_SCAN_ALREADY_RUNNING); + runloop_msg_queue_push(_msg, strlen(_msg), 1, 150, true, NULL, + MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_ERROR); + return 0; + } + if ((item = (task_dbscan_item_t*)calloc(1, sizeof(*item)))) + { + item->kind = TASK_DBSCAN_ITEM_REFRESH_ALL; + task_push_dbscan_chain(item, 1, NULL, + action_ok_playlist_refresh_all_cb, NULL); + } + return 0; +} + #ifdef HAVE_MIST static int action_ok_core_steam_install( const char *path, @@ -10031,6 +10079,7 @@ static int menu_cbs_init_bind_ok_compare_label(menu_file_list_cbs_t *cbs, {MENU_ENUM_LABEL_PLAYLIST_MANAGER_RESET_CORES, action_ok_playlist_reset_cores}, {MENU_ENUM_LABEL_PLAYLIST_MANAGER_CLEAN_PLAYLIST, action_ok_playlist_clean}, {MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_PLAYLIST, action_ok_playlist_refresh}, + {MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_ALL_PLAYLISTS, action_ok_playlist_refresh_all}, {MENU_ENUM_LABEL_ACCOUNTS_RETRO_ACHIEVEMENTS, action_ok_push_accounts_cheevos_list}, #ifdef HAVE_LAKKA {MENU_ENUM_LABEL_EJECT_DISC, action_ok_push_eject_disc}, diff --git a/menu/cbs/menu_cbs_scan.c b/menu/cbs/menu_cbs_scan.c index 517d1a60b0c9..0a4e53dd7896 100644 --- a/menu/cbs/menu_cbs_scan.c +++ b/menu/cbs/menu_cbs_scan.c @@ -50,6 +50,19 @@ void handle_dbscan_finished(retro_task_t *task, NULL, menu_st->userdata); } +/* A Scan Directory or Scan File would run with the playlist settings + * a running chain last loaded. */ +static bool action_scan_refused(void) +{ + const char *_msg; + if (!task_dbscan_chain_is_running()) + return false; + _msg = msg_hash_to_str(MSG_MANUAL_CONTENT_SCAN_ALREADY_RUNNING); + runloop_msg_queue_push(_msg, strlen(_msg), 1, 150, true, NULL, + MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_ERROR); + return true; +} + #ifdef HAVE_LIBRETRODB int action_scan_file(const char *path, const char *label, unsigned type, size_t idx) @@ -60,6 +73,9 @@ int action_scan_file(const char *path, char fullpath[PATH_MAX_LENGTH]; const char *menu_path = NULL; + if (action_scan_refused()) + return 0; + menu_entries_get_last_stack(&menu_path, NULL, NULL, NULL, NULL); #if TARGET_OS_IPHONE @@ -84,6 +100,9 @@ int action_scan_directory(const char *path, char fullpath[PATH_MAX_LENGTH]; const char *menu_path = NULL; + if (action_scan_refused()) + return 0; + menu_entries_get_last_stack(&menu_path, NULL, NULL, NULL, NULL); #if TARGET_OS_IPHONE diff --git a/menu/cbs/menu_cbs_sublabel.c b/menu/cbs/menu_cbs_sublabel.c index 618865afb28e..0dded411347e 100644 --- a/menu/cbs/menu_cbs_sublabel.c +++ b/menu/cbs/menu_cbs_sublabel.c @@ -2755,6 +2755,7 @@ int menu_cbs_init_bind_sublabel(menu_file_list_cbs_t *cbs, { MENU_ENUM_LABEL_PLAYLIST_MANAGER_SORT_MODE, MENU_ENUM_SUBLABEL_PLAYLIST_MANAGER_SORT_MODE }, { MENU_ENUM_LABEL_PLAYLIST_MANAGER_CLEAN_PLAYLIST, MENU_ENUM_SUBLABEL_PLAYLIST_MANAGER_CLEAN_PLAYLIST }, { MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_PLAYLIST, MENU_ENUM_SUBLABEL_PLAYLIST_MANAGER_REFRESH_PLAYLIST }, + { MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_ALL_PLAYLISTS, MENU_ENUM_SUBLABEL_PLAYLIST_MANAGER_REFRESH_ALL_PLAYLISTS }, { MENU_ENUM_LABEL_SAVE_AS_CONFIG, MENU_ENUM_SUBLABEL_SAVE_AS_CONFIG }, { MENU_ENUM_LABEL_SAVE_MAIN_CONFIG, MENU_ENUM_SUBLABEL_SAVE_MAIN_CONFIG }, { MENU_ENUM_LABEL_ONSCREEN_DISPLAY_SETTINGS, MENU_ENUM_SUBLABEL_ONSCREEN_DISPLAY_SETTINGS }, diff --git a/menu/drivers/materialui.c b/menu/drivers/materialui.c index ff1701f9a720..78d8ebef72b5 100644 --- a/menu/drivers/materialui.c +++ b/menu/drivers/materialui.c @@ -12746,6 +12746,7 @@ static void materialui_list_insert(void *userdata, || string_is_equal(label, MENU_ENUM_LABEL_UPDATE_GLSL_SHADERS_STR) || string_is_equal(label, MENU_ENUM_LABEL_UPDATE_SLANG_SHADERS_STR) || string_is_equal(label, MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_PLAYLIST_STR) + || string_is_equal(label, MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_ALL_PLAYLISTS_STR) ) { node->icon_texture_index = MUI_TEXTURE_UPDATER; diff --git a/menu/drivers/ozone.c b/menu/drivers/ozone.c index 642c96f4ff9d..b18aa113dfe1 100644 --- a/menu/drivers/ozone.c +++ b/menu/drivers/ozone.c @@ -2460,6 +2460,7 @@ static uintptr_t ozone_entries_icon_get_texture( case MENU_ENUM_LABEL_FRAME_TIME_COUNTER_SETTINGS: case MENU_ENUM_LABEL_PLAYLIST_MANAGER_CLEAN_PLAYLIST: case MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_PLAYLIST: + case MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_ALL_PLAYLISTS: case MENU_ENUM_LABEL_CLOUD_SYNC_SETTINGS: return icons_tex[OZONE_ENTRIES_ICONS_TEXTURE_RELOAD]; case MENU_ENUM_LABEL_SHUTDOWN: diff --git a/menu/drivers/xmb.c b/menu/drivers/xmb.c index 2e977e6c1316..e2dd1ab32bbb 100644 --- a/menu/drivers/xmb.c +++ b/menu/drivers/xmb.c @@ -4393,6 +4393,7 @@ static uintptr_t xmb_icon_get_id(xmb_handle_t *xmb, case MENU_ENUM_LABEL_FRAME_TIME_COUNTER_SETTINGS: case MENU_ENUM_LABEL_PLAYLIST_MANAGER_CLEAN_PLAYLIST: case MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_PLAYLIST: + case MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_ALL_PLAYLISTS: case MENU_ENUM_LABEL_CLOUD_SYNC_SETTINGS: return GFX_SURFACE_HANDLE(xmb->textures.list[XMB_TEXTURE_RELOAD]); case MENU_ENUM_LABEL_VRR_RUNLOOP_ENABLE: diff --git a/menu/menu_displaylist.c b/menu/menu_displaylist.c index c5febc3d8809..03bd693c83e7 100644 --- a/menu/menu_displaylist.c +++ b/menu/menu_displaylist.c @@ -6236,6 +6236,17 @@ static unsigned menu_displaylist_parse_playlist_manager_list( count++; #endif + /* None keeps the "No entries" fallback */ + if (count) + { + menu_entries_prepend(list, + msg_hash_to_str(MENU_ENUM_LABEL_VALUE_PLAYLIST_MANAGER_REFRESH_ALL_PLAYLISTS), + MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_ALL_PLAYLISTS_STR, + MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_ALL_PLAYLISTS, + MENU_SETTING_ACTION, 0, 0); + count++; + } + return count; } diff --git a/msg_hash.h b/msg_hash.h index 2dfe9347030d..0c4cc47dd8f6 100644 --- a/msg_hash.h +++ b/msg_hash.h @@ -29962,6 +29962,9 @@ enum msg_hash_enums MSG_PLAYLIST_MANAGER_REFRESH_INVALID_DAT_FILE, MSG_PLAYLIST_MANAGER_REFRESH_DAT_FILE_TOO_LARGE, + MENU_LABEL(PLAYLIST_MANAGER_REFRESH_ALL_PLAYLISTS), + MSG_PLAYLIST_MANAGER_REFRESH_ALL_DONE, + MENU_LABEL(CORE_UPDATER_SETTINGS), MENU_LABEL(SAVE_AS_CONFIG), MENU_LABEL(SAVE_MAIN_CONFIG), @@ -34351,6 +34354,7 @@ enum msg_hash_enums MSG_MANUAL_CONTENT_SCAN_IN_PROGRESS, MSG_MANUAL_CONTENT_SCAN_M3U_CLEANUP, MSG_MANUAL_CONTENT_SCAN_END, + MSG_MANUAL_CONTENT_SCAN_ALREADY_RUNNING, MSG_3DS_BOTTOM_MENU_DEFAULT, MSG_3DS_BOTTOM_MENU_ASSET_NOT_FOUND, diff --git a/msg_hash_lbl_str.h b/msg_hash_lbl_str.h index 3d46daa8ba41..214bda861d80 100644 --- a/msg_hash_lbl_str.h +++ b/msg_hash_lbl_str.h @@ -486,6 +486,7 @@ extern const struct msg_hash_us_lbl_blob_s msg_hash_us_lbl_blob; #define MENU_ENUM_LABEL_PLAYLIST_MANAGER_SORT_MODE_STR (msg_hash_us_lbl_blob.m_MENU_ENUM_LABEL_PLAYLIST_MANAGER_SORT_MODE) #define MENU_ENUM_LABEL_PLAYLIST_MANAGER_CLEAN_PLAYLIST_STR (msg_hash_us_lbl_blob.m_MENU_ENUM_LABEL_PLAYLIST_MANAGER_CLEAN_PLAYLIST) #define MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_PLAYLIST_STR (msg_hash_us_lbl_blob.m_MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_PLAYLIST) +#define MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_ALL_PLAYLISTS_STR (msg_hash_us_lbl_blob.m_MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_ALL_PLAYLISTS) #define MENU_ENUM_LABEL_PLAYLIST_SETTINGS_BEGIN_STR (msg_hash_us_lbl_blob.m_MENU_ENUM_LABEL_PLAYLIST_SETTINGS_BEGIN) #define MENU_ENUM_LABEL_RDB_ENTRY_STR (msg_hash_us_lbl_blob.m_MENU_ENUM_LABEL_RDB_ENTRY) #define MENU_ENUM_LABEL_RDB_ENTRY_DETAIL_STR (msg_hash_us_lbl_blob.m_MENU_ENUM_LABEL_RDB_ENTRY_DETAIL) @@ -2238,6 +2239,7 @@ extern const struct msg_hash_us_lbl_blob_s msg_hash_us_lbl_blob; #define MENU_ENUM_LABEL_PLAYLIST_MANAGER_SORT_MODE_STR "playlist_manager_sort_mode" #define MENU_ENUM_LABEL_PLAYLIST_MANAGER_CLEAN_PLAYLIST_STR "playlist_manager_clean_playlist" #define MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_PLAYLIST_STR "playlist_manager_refresh_playlist" +#define MENU_ENUM_LABEL_PLAYLIST_MANAGER_REFRESH_ALL_PLAYLISTS_STR "playlist_manager_refresh_all_playlists" #define MENU_ENUM_LABEL_PLAYLIST_SETTINGS_BEGIN_STR "playlist_settings_begin" #define MENU_ENUM_LABEL_RDB_ENTRY_STR "rdb_entry" #define MENU_ENUM_LABEL_RDB_ENTRY_DETAIL_STR "rdb_entry_detail" diff --git a/samples/tasks/database/database_scan_test.c b/samples/tasks/database/database_scan_test.c index 2a45c4de49f1..291d7ab3121b 100644 --- a/samples/tasks/database/database_scan_test.c +++ b/samples/tasks/database/database_scan_test.c @@ -139,10 +139,36 @@ static void end_step_hold(void) retro_atomic_store_release_int(&end_readers_out, n ? 0 : 1); } +/* The English rows, for the messages a check compares whole. */ +static const struct +{ + enum msg_hash_enums id; + const char *str; +} msg_hash_us_rows[] = { +#undef MSG_HASH +#define MSG_HASH(Id, str) { Id, str }, +#include "../../../intl/msg_hash_us.h" +#undef MSG_HASH +}; + +static const char *msg_hash_us_row(enum msg_hash_enums msg) +{ + size_t i; + + for (i = 0; i < sizeof(msg_hash_us_rows) / sizeof(msg_hash_us_rows[0]); + i++) + if (msg_hash_us_rows[i].id == msg) + return msg_hash_us_rows[i].str; + return ""; +} + const char *msg_hash_to_str_us(enum msg_hash_enums msg) { if (msg == MSG_SCANNING_OF_DIRECTORY_FINISHED && end_hold) end_step_hold(); + if ( msg == MSG_PLAYLIST_MANAGER_REFRESH_ALL_DONE + || msg == MSG_MANUAL_CONTENT_SCAN_ALREADY_RUNNING) + return msg_hash_us_row(msg); if (msg == MSG_MANUAL_CONTENT_SCAN_INVALID_CONTENT) return "invalid content"; if (msg == MSG_SCANNING_NO_DATABASE) @@ -9233,7 +9259,8 @@ static void chain_lane(const char *root, const char *pl_dir, char detail[160]; const char *q = threaded ? "(threaded)" : "(regular)"; task_dbscan_item_t *items; - bool pushed, refused, captured, back; + bool pushed, refused, captured, back, running; + bool scan_refused; sprintf(a, "%s/chain_a", root); sprintf(b, "%s/chain_b", root); @@ -9338,13 +9365,17 @@ static void chain_lane(const char *root, const char *pl_dir, && chain_totals.ran == 1 && chain_totals.cancelled, what, detail); pushed = chain_push(chain_items(1, a, NULL), 1); + running = task_dbscan_chain_is_running(); refused = !task_push_dbscan_chain(chain_items(1, b, NULL), 1, chain_item_cb, chain_done_cb, &chain_seen); + scan_refused = !task_push_dbscan(b, scan_cb, NULL); chain_wait(false); sprintf(what, "chain %s: a second chain refused", q); sprintf(detail, "%u items, done %u", chain_seen, chain_dones); check( pushed && refused && chain_seen == 1 && chain_dones == 1 && chain_saw(0, a, TASK_DBSCAN_RAN, 0), what, detail); + sprintf(what, "chain %s: running from the push to the end", q); + check(running && !task_dbscan_chain_is_running(), what, ""); /* Held, so the scan cannot end before the chain is pushed */ slow_reset(sz_a, 0); @@ -9368,6 +9399,8 @@ static void chain_lane(const char *root, const char *pl_dir, sprintf(what, "chain %s: refused while a manual scan runs", q); check( pushed && refused && scan_completed && !chain_seen && !chain_dones, what, refused ? "refused" : "pushed"); + sprintf(what, "chain %s: no task_push_dbscan while it runs", q); + check(scan_refused && pushed, what, scan_refused ? "refused" : "pushed"); manual_content_scan_reset_settings(); strlcpy(config_get_ptr()->paths.directory_playlist, pl_dir, @@ -9949,6 +9982,111 @@ static void refresh_lane(const char *root, const char *pl_dir, sizeof(config_get_ptr()->paths.directory_playlist)); } +/* The summary the menu's Refresh All Playlists shows. */ +static char menu_all_msg[128]; + +static void menu_all_done_cb(const task_dbscan_chain_result_t *res, + void *user_data) +{ + if (user_data) + chain_ud_ok = false; + snprintf(menu_all_msg, sizeof(menu_all_msg), + msg_hash_to_str(MSG_PLAYLIST_MANAGER_REFRESH_ALL_DONE), + res->ran, res->skipped, res->failed); + chain_totals = *res; + chain_dones++; +} + +/* The one item the menu pushes. */ +static task_dbscan_item_t *menu_all_item(void) +{ + task_dbscan_item_t *item = (task_dbscan_item_t*)calloc(1, + sizeof(*item)); + + if (item) + item->kind = TASK_DBSCAN_ITEM_REFRESH_ALL; + return item; +} + +/* Refresh All Playlists as the menu pushes it, over A refreshable, B + * with its folder gone, C without settings and E with a missing core: + * the English summary, nothing printed, and a second press while it + * runs refused with the English message. */ +static void menu_refresh_all_lane(const char *root, const char *pl_dir, + uint32_t crc_a, uint32_t sz_a, bool threaded) +{ + char pl[512], a[512], gone[512], out[1024], err[1024]; + char p[1024], stale[1024], what[128]; + const char *q = threaded ? "(threaded)" : "(regular)"; + const char *busy; + bool captured, pushed, refused; + + sprintf(pl, "%s/menu_all_pl", root); + sprintf(a, "%s/menu_all_a", root); + sprintf(gone, "%s/menu_all_gone", root); + sprintf(out, "%s/menu_all_out.txt", root); + sprintf(err, "%s/menu_all_err.txt", root); + sprintf(stale, "%s/00_gone.bin", a); + path_mkdir(pl); + path_mkdir(a); + delete_playlists(pl); + sprintf(p, "%s/01_alpha.bin", a); + if (!write_content(p, crc_a, sz_a)) + goto fixture; + sprintf(p, "%s/a_ok.lpl", pl); + if (!refresh_write(p, a, NULL, stale)) + goto fixture; + sprintf(p, "%s/b_gone.lpl", pl); + if (!refresh_write(p, gone, NULL, stale)) + goto fixture; + sprintf(p, "%s/c_none.lpl", pl); + if (!refresh_write(p, NULL, NULL, stale)) + goto fixture; + sprintf(p, "%s/e_core.lpl", pl); + if (!refresh_write(p, a, "No Such Core", stale)) + goto fixture; + strlcpy(config_get_ptr()->paths.directory_playlist, pl, + sizeof(config_get_ptr()->paths.directory_playlist)); + use_queue(threaded); + + chain_seen = 0; + chain_dones = 0; + chain_ud_ok = true; + menu_all_msg[0] = '\0'; + cli_output = false; + captured = capture_begin(out, err); + pushed = task_push_dbscan_chain(menu_all_item(), 1, NULL, + menu_all_done_cb, NULL); + refused = !task_push_dbscan_chain(menu_all_item(), 1, NULL, + menu_all_done_cb, NULL); + chain_wait(false); + if (captured) + capture_end(); + sprintf(what, "menu refresh all %s: the summary", q); + check( pushed && chain_dones == 1 && chain_ud_ok + && !strcmp(menu_all_msg, + "Playlists refreshed: 1, skipped: 1, failed: 1"), + what, menu_all_msg); + busy = msg_hash_to_str(MSG_MANUAL_CONTENT_SCAN_ALREADY_RUNNING); + sprintf(what, "menu refresh all %s: a second press refused", q); + check( refused && chain_dones == 1 + && !strcmp(busy, "A content scan is already running."), + what, busy); + sprintf(what, "menu refresh all %s: nothing printed", q); + check( captured && path_get_size(out) == 0 + && path_get_size(err) == 0, what, err); + + manual_content_scan_reset_settings(); + strlcpy(config_get_ptr()->paths.directory_playlist, pl_dir, + sizeof(config_get_ptr()->paths.directory_playlist)); + return; + +fixture: + check(0, "fixture", "could not write the menu refresh-all lane"); + strlcpy(config_get_ptr()->paths.directory_playlist, pl_dir, + sizeof(config_get_ptr()->paths.directory_playlist)); +} + int main(int argc, char **argv) { const char *root = (argc > 1) ? argv[1] : "/tmp"; @@ -11038,6 +11176,8 @@ int main(int argc, char **argv) chain_lane(root, pl_dir, crc_a, sz_a, true); refresh_lane(root, pl_dir, crc_a, sz_a, false); refresh_lane(root, pl_dir, crc_a, sz_a, true); + menu_refresh_all_lane(root, pl_dir, crc_a, sz_a, false); + menu_refresh_all_lane(root, pl_dir, crc_a, sz_a, true); { char reader_db[512]; diff --git a/tasks/task_database.c b/tasks/task_database.c index 47349dc20a51..5a68ef32bd16 100644 --- a/tasks/task_database.c +++ b/tasks/task_database.c @@ -4214,6 +4214,9 @@ static bool manual_scan_end_flush_tick( bool task_push_dbscan(const char *fullpath, retro_task_callback_t cb, void *user_data) { + /* It would run with the settings the chain last loaded. */ + if (task_dbscan_chain_is_running()) + return false; manual_content_scan_set_menu_content_dir(fullpath); return task_push_manual_content_scan(false, cb, user_data); } @@ -6783,18 +6786,24 @@ static bool dbscan_chain_scan_finder(retro_task_t *task, void *user_data) && !(task_get_flags(task) & RETRO_TASK_FLG_FINISHED); } +bool task_dbscan_chain_can_start(void) +{ + task_finder_data_t find_data; + + find_data.func = dbscan_chain_scan_finder; + find_data.userdata = NULL; + return !dbscan_chain_running && !task_queue_find(&find_data); +} + bool task_push_dbscan_chain(task_dbscan_item_t *items, size_t count, task_dbscan_item_cb_t item_cb, task_dbscan_chain_cb_t done_cb, void *user_data) { - task_finder_data_t find_data; dbscan_chain_t *chain = NULL; dbscan_prep_t *prep = NULL; settings_t *settings = config_get_ptr(); - find_data.func = dbscan_chain_scan_finder; - find_data.userdata = NULL; - if (dbscan_chain_running || task_queue_find(&find_data)) + if (!task_dbscan_chain_can_start()) goto error; if ( !(chain = (dbscan_chain_t*)calloc(1, sizeof(*chain))) || !(prep = (dbscan_prep_t*)calloc(1, sizeof(*prep)))) @@ -6830,3 +6839,8 @@ bool task_push_dbscan_chain(task_dbscan_item_t *items, size_t count, dbscan_chain_free_items(items, count); return false; } + +bool task_dbscan_chain_is_running(void) +{ + return dbscan_chain_running != NULL; +} diff --git a/tasks/tasks_internal.h b/tasks/tasks_internal.h index ae5a9b9d33ae..9f920c1196c4 100644 --- a/tasks/tasks_internal.h +++ b/tasks/tasks_internal.h @@ -351,7 +351,7 @@ typedef struct #ifdef HAVE_LIBRETRODB /* Scans @fullpath, a directory or a single file, against the content * databases; the database and playlist directories come from the - * settings. @cb gets @user_data. */ + * settings. @cb gets @user_data. False while a chain runs. */ bool task_push_dbscan(const char *fullpath, retro_task_callback_t cb, void *user_data); #endif @@ -405,6 +405,13 @@ bool task_push_dbscan_chain(task_dbscan_item_t *items, size_t count, task_dbscan_item_cb_t item_cb, task_dbscan_chain_cb_t done_cb, void *user_data); +/* Main thread only. */ +bool task_dbscan_chain_is_running(void); + +/* False while task_push_dbscan_chain() would refuse: a chain or a + * manual scan runs. Main thread only. */ +bool task_dbscan_chain_can_start(void); + bool task_push_manual_content_scan( bool do_menu_refresh, retro_task_callback_t user_cb, void *user_data); From f6da0e15e8a0a37b292fd07f118b582a6227ca24 Mon Sep 17 00:00:00 2001 From: davey Date: Mon, 2 Nov 2026 02:29:49 -0800 Subject: [PATCH 17/21] docs: --scan, refresh and the Manual Scan options in the man page --help lists the options but not what a script needs: the order the scans run in, the exit status and what the options set. The man page now has them, next to --cloudsync, and CHANGES.md has the entry. --- CHANGES.md | 1 + docs/retroarch.6 | 62 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 7d208cc04c23..82b19c40fcda 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -145,6 +145,7 @@ - SCAN: Interpret M3U files during scanning - SCAN: Merge manual and automatic scan into a more flexible setup - SCAN: Improve GDI content recognition +- SCAN: --scan runs every path and exits with what the scans did; Manual Scan options, --scan-refresh and --scan-refresh-all on the command line; Refresh All Playlists in Manage Playlists - SHADER: Display on-screen error when preset load fails - SHADER: Add gyroscope / accelerometer uniforms for shaders - SHADER: Add aggressive shader caching to slang shader backend diff --git a/docs/retroarch.6 b/docs/retroarch.6 index 342756d5deb9..1e44693c7f07 100644 --- a/docs/retroarch.6 +++ b/docs/retroarch.6 @@ -65,6 +65,68 @@ Prints available features compiled into RetroArch, then exits. \fB--cloudsync\fR Runs a cloud sync with the configured cloud sync driver, then exits. +.TP +\fB--scan PATH\fR +Scans the folder or file PATH for content and adds it to playlists, as Manual Scan does. +Can be given more than once. + +.TP +\fB--scan-refresh NAME|PATH\fR +Rescans one playlist with the scan settings it saved, as Refresh Playlist does. +NAME matches a playlist's name exactly first, then ignoring case. PATH must be in the playlist folder. +Like Refresh Playlist, it drops entries whose files are gone or whose extension the saved settings no longer include. +A playlist whose saved content folder is missing is skipped instead, so an unplugged drive does not empty it. + +.TP +\fB--scan-refresh-all\fR +Does --scan-refresh for every playlist that has saved scan settings, as Refresh All Playlists does. + +.IP +These run in the order given, and one that fails does not stop the rest. +Each prints a totals line on stdout, or its error on stderr. +Without --menu, RetroArch exits when they are done and does not load content given on the command line. +The exit status is then 1 if any failed, 0 if any added an entry, and 2 if none added anything; a skipped playlist does not count. +With --menu, they run while the menu does. +They cannot be combined with --cloudsync or --command. + +.IP +The options below set the Manual Scan settings of the --scan, --scan-refresh or --scan-refresh-all before them. +On a refresh they override the saved settings, and the refresh saves them. +A --scan without any is Fully Automatic. + +.TP +\fB--scan-playlist NAME\fR +Sets Target Playlist: a database's name, or a custom playlist name. Not allowed on a refresh. + +.TP +\fB--scan-database NAME|first\fR +Sets Database To Match: the database NAME, or the first database that matches. + +.TP +\fB--scan-match strict|loose|none\fR +Sets Database Check. + +.TP +\fB--scan-dat FILE\fR +Sets Arcade DAT File. Matching is strict unless --scan-match loose is given. + +.TP +\fB--scan-core CORE\fR +Sets Default Core: a core's path, its file name (with or without _libretro and the extension) or its display name. + +.TP +\fB--scan-exts EXTS\fR +Sets File Extensions, separated by commas or spaces. + +.TP +\fB--scan-no-recursive, --scan-no-archives\fR +Turn off Scan Recursively and Scan Inside Archives. + +.TP +\fB--scan-overwrite, --scan-validate, --scan-omit-db-ref\fR +Turn on Overwrite Existing Playlist, Validate Existing Entries and Skip Database References From Playlist. +They act only when the scan has one playlist, and --scan-overwrite only on a folder; otherwise they print a warning and are ignored. + .TP \fB-L PATH, --libretro PATH\fR Path to a libretro implementation which is to be used. From 53f8ce1bcb8399fc7bffd204fe32a090bd12e133 Mon Sep 17 00:00:00 2001 From: LibretroAdmin Date: Wed, 4 Nov 2026 18:38:25 +0100 Subject: [PATCH 18/21] scan chain: writes from workers reach the playlist cache by note A playlist written by a task - the chain's prep parse, a scan, a playlist-manager task, whatever thread it runs on - leaves a note on a lock-free stack instead of reading the cached playlist; the main thread applies the notes when it next decides whether the cache may stand in, so no worker reads playlist_cached while the main thread frees or installs it. Without threads or pointer atomics a write applies at once. The chain's prep tasks set their handlers by name, and thread_read_audit.py fails on a handler set from a retro_task_handler_t parameter, which would hide every handler passed in. The chain puts the Manual Scan settings it found back when it ends, and is no longer running when its done callback runs, so that callback may start another. task_push_dbscan() says why it refuses, so the menu and the companions show the same reason, and the companions' status line takes it from companion_core. --scan and --scan-refresh set the verbosity override, so a config with logging off cannot detach the console the totals go to, and refuse a path too long for the chain's buffers. The Refresh All notification passes the length written, and fill_pathname_expand_special() stops at the end of a buffer shorter than the home or application directory. Lanes: playlist_parity_test writes the cached file from a worker, once at the same size and then while the main thread reuses and frees the cache; database_scan_test checks the refusal message, the restored settings and a done callback that starts the next chain; fill_pathname_test expands into a buffer shorter than the home directory. --- libretro-common/file/file_path.c | 12 +- .../file/file_path/fill_pathname_test.c | 30 +++++ manual_content_scan.c | 50 ++++++++ manual_content_scan.h | 6 + menu/cbs/menu_cbs_ok.c | 28 +---- menu/cbs/menu_cbs_scan.c | 19 --- playlist.c | 111 +++++++++++++----- retroarch.c | 13 ++ samples/playlist/playlist_parity_test.c | 97 +++++++++++++++ samples/tasks/database/database_scan_test.c | 62 ++++++++++ tasks/task_database.c | 56 +++++++-- tasks/tasks_internal.h | 5 + tools/thread_read_audit.py | 34 +++++- ui/companion/companion_core.c | 9 ++ ui/companion/companion_core.h | 2 + ui/companion/test/companion_core_stubs.c | 2 + ui/drivers/ui_cocoa_companion.m | 2 +- ui/drivers/ui_win32_companion.c | 2 +- 18 files changed, 454 insertions(+), 86 deletions(-) diff --git a/libretro-common/file/file_path.c b/libretro-common/file/file_path.c index 88734e0d8e26..9bc43fdbfa94 100644 --- a/libretro-common/file/file_path.c +++ b/libretro-common/file/file_path.c @@ -1120,6 +1120,7 @@ size_t fill_pathname_join_delim(char *s, const char *dir, size_t fill_pathname_expand_special(char *s, const char *in_path, size_t len) { #if !defined(RARCH_CONSOLE) && defined(RARCH_INTERNAL) + char *start = s; if (in_path[0] == '~' || in_path[0] == ':') { char app_dir[DIR_MAX_LENGTH]; @@ -1134,8 +1135,13 @@ size_t fill_pathname_expand_special(char *s, const char *in_path, size_t len) if (*app_dir) { + /* strlcpy() reports the source's length whether or not it + * fit: a directory longer than @s leaves the result cut + * there, with nothing appended past the end. */ size_t _len = strlcpy(s, app_dir, len); + if (_len >= len) + return len ? len - 1 : 0; s += _len; len -= _len; @@ -1143,6 +1149,8 @@ size_t fill_pathname_expand_special(char *s, const char *in_path, size_t len) { _len = strlcpy(s, PATH_DEFAULT_SLASH(), len); + if (_len >= len) + return (size_t)(s - start) + (len ? len - 1 : 0); s += _len; len -= _len; } @@ -1152,8 +1160,10 @@ size_t fill_pathname_expand_special(char *s, const char *in_path, size_t len) in_path++; } } -#endif + return (size_t)(s - start) + strlcpy(s, in_path, len); +#else return strlcpy(s, in_path, len); +#endif } size_t fill_pathname_abbreviate_special(char *s, diff --git a/libretro-common/samples/file/file_path/fill_pathname_test.c b/libretro-common/samples/file/file_path/fill_pathname_test.c index b481f76b0df7..deb4de036d0a 100644 --- a/libretro-common/samples/file/file_path/fill_pathname_test.c +++ b/libretro-common/samples/file/file_path/fill_pathname_test.c @@ -370,6 +370,35 @@ static void test_expand_special_home(void) } } +/* A home directory longer than the buffer: the expansion is cut at the + * buffer's end and nothing is written past it. The buffer is a heap + * block of exactly its size, so ASan sees a write past the end. */ +static void test_expand_special_truncation(void) +{ + const size_t len = 8; + char *out = (char*)malloc(len); + size_t got; + + if (!out) + { + printf("[FAILED] expand truncation: out of memory\n"); + failures++; + return; + } + set_home("/a-home-directory-longer-than-the-buffer"); + got = fill_pathname_expand_special(out, "~/file", len); + if (got != len - 1 || strlen(out) != len - 1 + || strncmp(out, "/a-home", len - 1)) + { + printf("[FAILED] expand truncation gave \"%s\" (%u)\n", + out, (unsigned)got); + failures++; + } + else + printf("[SUCCESS] expand truncation\n"); + free(out); +} + int main(void) { /* Documented semantics. */ @@ -391,6 +420,7 @@ int main(void) test_join_truncation(); test_abbreviated_or_relative_long_join(); test_expand_special_home(); + test_expand_special_truncation(); if (failures) { diff --git a/manual_content_scan.c b/manual_content_scan.c index fe3ab70c2fc6..094a40009aef 100644 --- a/manual_content_scan.c +++ b/manual_content_scan.c @@ -380,6 +380,56 @@ void manual_content_scan_reset_settings(void) scan_core_name[0] = '\0'; } +struct manual_content_scan_saved +{ + scan_settings_t settings; + char core_path[PATH_MAX_LENGTH]; + char file_exts_core[PATH_MAX_LENGTH]; + char file_exts_custom[PATH_MAX_LENGTH]; + char dat_file_path[PATH_MAX_LENGTH]; + char content_dir[PATH_MAX_LENGTH]; + char system_name_content_dir[DIR_MAX_LENGTH]; + char system_name_database[NAME_MAX_LENGTH]; + char system_name_custom[NAME_MAX_LENGTH]; + char core_name[NAME_MAX_LENGTH]; +}; + +manual_content_scan_saved_t *manual_content_scan_save_settings(void) +{ + manual_content_scan_saved_t *sv = (manual_content_scan_saved_t*) + malloc(sizeof(*sv)); + if (!sv) + return NULL; + sv->settings = scan_settings; + memcpy(sv->core_path, scan_core_path, sizeof(sv->core_path)); + memcpy(sv->file_exts_core, scan_file_exts_core, sizeof(sv->file_exts_core)); + memcpy(sv->file_exts_custom, scan_file_exts_custom, sizeof(sv->file_exts_custom)); + memcpy(sv->dat_file_path, scan_dat_file_path, sizeof(sv->dat_file_path)); + memcpy(sv->content_dir, scan_content_dir, sizeof(sv->content_dir)); + memcpy(sv->system_name_content_dir, scan_system_name_content_dir, sizeof(sv->system_name_content_dir)); + memcpy(sv->system_name_database, scan_system_name_database, sizeof(sv->system_name_database)); + memcpy(sv->system_name_custom, scan_system_name_custom, sizeof(sv->system_name_custom)); + memcpy(sv->core_name, scan_core_name, sizeof(sv->core_name)); + return sv; +} + +void manual_content_scan_restore_settings(manual_content_scan_saved_t *sv) +{ + if (!sv) + return; + scan_settings = sv->settings; + memcpy(scan_core_path, sv->core_path, sizeof(scan_core_path)); + memcpy(scan_file_exts_core, sv->file_exts_core, sizeof(scan_file_exts_core)); + memcpy(scan_file_exts_custom, sv->file_exts_custom, sizeof(scan_file_exts_custom)); + memcpy(scan_dat_file_path, sv->dat_file_path, sizeof(scan_dat_file_path)); + memcpy(scan_content_dir, sv->content_dir, sizeof(scan_content_dir)); + memcpy(scan_system_name_content_dir, sv->system_name_content_dir, sizeof(scan_system_name_content_dir)); + memcpy(scan_system_name_database, sv->system_name_database, sizeof(scan_system_name_database)); + memcpy(scan_system_name_custom, sv->system_name_custom, sizeof(scan_system_name_custom)); + memcpy(scan_core_name, sv->core_name, sizeof(scan_core_name)); + free(sv); +} + /* Sets content directory for next manual scan * operation. * Returns true if content directory is valid. */ diff --git a/manual_content_scan.h b/manual_content_scan.h index 8e8614467981..002dee24845d 100644 --- a/manual_content_scan.h +++ b/manual_content_scan.h @@ -263,6 +263,12 @@ enum manual_content_scan_dat_file_path_status /* Sets every setting back to its default */ void manual_content_scan_reset_settings(void); +/* The Manual Scan settings as they are, for a scan chain to put back + * once it has run its own: NULL out of memory. Restoring frees it. */ +typedef struct manual_content_scan_saved manual_content_scan_saved_t; +manual_content_scan_saved_t *manual_content_scan_save_settings(void); +void manual_content_scan_restore_settings(manual_content_scan_saved_t *saved); + /* Sets content directory for next manual scan * operation. * Returns true if content directory is valid. */ diff --git a/menu/cbs/menu_cbs_ok.c b/menu/cbs/menu_cbs_ok.c index f09a0b8eb6c9..7ca487568c46 100644 --- a/menu/cbs/menu_cbs_ok.c +++ b/menu/cbs/menu_cbs_ok.c @@ -9123,14 +9123,8 @@ static int action_ok_manual_content_scan_start(const char *path, { playlist_config_t playlist_config; settings_t *settings = config_get_ptr(); - if (task_dbscan_chain_is_running()) - { - const char *_msg = msg_hash_to_str( - MSG_MANUAL_CONTENT_SCAN_ALREADY_RUNNING); - runloop_msg_queue_push(_msg, strlen(_msg), 1, 150, true, NULL, - MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_ERROR); + if (task_dbscan_chain_refuse()) return 0; - } /* Note: playlist_config.path will set by the * task itself */ playlist_config.capacity = COLLECTION_SIZE; @@ -9607,14 +9601,8 @@ static int action_ok_playlist_refresh(const char *path, playlist_config = playlist_get_config(playlist); if (!playlist_config || !*playlist_config->path) return -1; - if (task_dbscan_chain_is_running()) - { - const char *_msg = msg_hash_to_str( - MSG_MANUAL_CONTENT_SCAN_ALREADY_RUNNING); - runloop_msg_queue_push(_msg, strlen(_msg), 1, 150, true, NULL, - MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_ERROR); + if (task_dbscan_chain_refuse()) return 0; - } stat = manual_content_scan_set_menu_from_playlist(playlist, settings->paths.path_content_database, settings->bools.show_hidden_files); @@ -9708,9 +9696,11 @@ static void action_ok_playlist_refresh_all_cb( const task_dbscan_chain_result_t *res, void *user_data) { char msg[128]; - size_t _len = snprintf(msg, sizeof(msg), + int n = snprintf(msg, sizeof(msg), msg_hash_to_str(MSG_PLAYLIST_MANAGER_REFRESH_ALL_DONE), res->ran, res->skipped, res->failed); + /* the length written, not the one a translation asked for */ + size_t _len = n < 0 ? 0 : (size_t)n >= sizeof(msg) ? sizeof(msg) - 1 : (size_t)n; runloop_msg_queue_push(msg, _len, 1, 180, true, NULL, MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_INFO); } @@ -9719,14 +9709,8 @@ static int action_ok_playlist_refresh_all(const char *path, const char *label, unsigned type, size_t idx, size_t entry_idx) { task_dbscan_item_t *item = NULL; - if (!task_dbscan_chain_can_start()) - { - const char *_msg = msg_hash_to_str( - MSG_MANUAL_CONTENT_SCAN_ALREADY_RUNNING); - runloop_msg_queue_push(_msg, strlen(_msg), 1, 150, true, NULL, - MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_ERROR); + if (task_dbscan_chain_refuse_start()) return 0; - } if ((item = (task_dbscan_item_t*)calloc(1, sizeof(*item)))) { item->kind = TASK_DBSCAN_ITEM_REFRESH_ALL; diff --git a/menu/cbs/menu_cbs_scan.c b/menu/cbs/menu_cbs_scan.c index 0a4e53dd7896..517d1a60b0c9 100644 --- a/menu/cbs/menu_cbs_scan.c +++ b/menu/cbs/menu_cbs_scan.c @@ -50,19 +50,6 @@ void handle_dbscan_finished(retro_task_t *task, NULL, menu_st->userdata); } -/* A Scan Directory or Scan File would run with the playlist settings - * a running chain last loaded. */ -static bool action_scan_refused(void) -{ - const char *_msg; - if (!task_dbscan_chain_is_running()) - return false; - _msg = msg_hash_to_str(MSG_MANUAL_CONTENT_SCAN_ALREADY_RUNNING); - runloop_msg_queue_push(_msg, strlen(_msg), 1, 150, true, NULL, - MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_ERROR); - return true; -} - #ifdef HAVE_LIBRETRODB int action_scan_file(const char *path, const char *label, unsigned type, size_t idx) @@ -73,9 +60,6 @@ int action_scan_file(const char *path, char fullpath[PATH_MAX_LENGTH]; const char *menu_path = NULL; - if (action_scan_refused()) - return 0; - menu_entries_get_last_stack(&menu_path, NULL, NULL, NULL, NULL); #if TARGET_OS_IPHONE @@ -100,9 +84,6 @@ int action_scan_directory(const char *path, char fullpath[PATH_MAX_LENGTH]; const char *menu_path = NULL; - if (action_scan_refused()) - return 0; - menu_entries_get_last_stack(&menu_path, NULL, NULL, NULL, NULL); #if TARGET_OS_IPHONE diff --git a/playlist.c b/playlist.c index 5d4ece7a0d0f..2931e5b5286e 100644 --- a/playlist.c +++ b/playlist.c @@ -32,6 +32,8 @@ #include #include #include +#include +#include #include "playlist.h" #include "verbosity.h" @@ -156,9 +158,27 @@ typedef struct static playlist_t *playlist_cached = NULL; /* Set when the cached playlist's file has been rewritten by something * else, so the next request re-reads it. See - * playlist_cached_after_write for why this is a flag and not a free. */ + * playlist_cached_note_write for why this is a flag and not a free. */ static bool playlist_cached_stale = false; +/* A write reaches the cache through a note, from whatever thread made + * it: the main thread applies the notes when it next decides whether + * the cache may stand in, so no writer reads or writes playlist_cached. + * Where the pointer atomics or threads are missing the write applies + * at once, as every write then comes from the one thread. */ +#if defined(HAVE_THREADS) && defined(RETRO_ATOMIC_HAS_PTR) +#define PLAYLIST_WRITE_NOTES 1 +typedef struct playlist_write_note +{ + mpsc_stack_node_t node; + const playlist_t *written; /* compared, never dereferenced */ + int64_t size; /* the file's size after the write */ + char path[1]; /* allocated to fit */ +} playlist_write_note_t; + +static mpsc_stack_t playlist_write_notes; +#endif + typedef int (playlist_sort_fun_t)( const struct playlist_entry *a, const struct playlist_entry *b); @@ -2168,40 +2188,72 @@ void playlist_write_runtime_file(playlist_t *playlist) } } -/* A playlist file has just been written. If the cached playlist is a - * different object reading the same path, its contents are now stale - - * a scan or a playlist-manager task writing the file the menu happens - * to have open is the ordinary way this happens. Drop it rather than - * let a later reuse serve what is on disk no longer. When the cached - * playlist is the one that was written, it is still authoritative; only - * its size stamp needs to catch up. */ -static void playlist_cached_after_write(playlist_t *written) +/* What one write means for the cache: a write of the cached playlist + * itself moves its size stamp; any other write to its file leaves it + * stale, flagged rather than freed so nothing holding the pointer + * playlist_get_cached returned loses it (the next playlist_init_cached + * re-reads, at a point where nothing holds the old one). Main thread. */ +static void playlist_cached_note_write(const playlist_t *written, + const char *path, int64_t size) { - if (!playlist_cached || !written || !*written->config.path) + if (!playlist_cached || !string_is_equal(playlist_cached->config.path, path)) return; if (playlist_cached == written) + playlist_cached->file_size = size; + else + playlist_cached_stale = true; +} + +/* The notes since the last look, newest first: only the newest write of + * the cached playlist sets its stamp. Main thread. */ +static void playlist_cached_apply_writes(void) +{ +#ifdef PLAYLIST_WRITE_NOTES + bool sized = false; + mpsc_stack_node_t *node = mpsc_stack_drain(&playlist_write_notes); + while (node) { - playlist_cached->file_size = path_get_size(written->config.path); - return; + mpsc_stack_node_t *next = node->next; + playlist_write_note_t *n = (playlist_write_note_t*)node; + if (playlist_cached && n->written == playlist_cached) + { + if (!sized) + playlist_cached_note_write(n->written, n->path, n->size); + sized = true; + } + else + playlist_cached_note_write(n->written, n->path, n->size); + free(n); + node = next; } - if (string_is_equal(playlist_cached->config.path, written->config.path)) +#endif +} + +/* A playlist file has just been written, possibly by a task on a + * worker. A scan or a playlist-manager task writing the file the menu + * has open is the ordinary way the cache goes stale. */ +static void playlist_cached_after_write(playlist_t *written) +{ +#ifdef PLAYLIST_WRITE_NOTES + size_t len; + playlist_write_note_t *n; + if (!written || !*written->config.path) + return; + len = strlen(written->config.path); + if (!(n = (playlist_write_note_t*)malloc(sizeof(*n) + len))) { - /* Mark it, do not free it. - * - * This runs from whatever happened to write the file, and the - * commonest case is the history list: launching an entry pushes - * it onto g_defaults.content_history, which is a different - * playlist_t for the same path as the one the menu is displaying - * from. Freeing here would pull that object out from under - * every menu_displaylist caller still holding what - * playlist_get_cached returned - a use after free whose symptoms - * appear later and somewhere else entirely. - * - * The staleness is what matters, not the memory: flagging it - * makes the next playlist_init_cached re-read, which is a point - * where nothing is holding the old pointer. */ - playlist_cached_stale = true; + /* no note to leave: a later size check still catches a change */ + return; } + n->written = written; + n->size = path_get_size(written->config.path); + memcpy(n->path, written->config.path, len + 1); + mpsc_stack_push(&playlist_write_notes, &n->node); +#else + if (written && *written->config.path) + playlist_cached_note_write(written, written->config.path, + path_get_size(written->config.path)); +#endif } void playlist_write_file(playlist_t *playlist) @@ -3950,6 +4002,8 @@ void playlist_parse_abort(playlist_parse_t *p) void playlist_free_cached(void) { + /* what was written since no longer has a cache to tell */ + playlist_cached_apply_writes(); if (playlist_cached && !(playlist_cached->flags & CNT_PLAYLIST_FLG_CACHED_EXT)) playlist_free(playlist_cached); playlist_cached = NULL; @@ -3980,6 +4034,7 @@ playlist_t *playlist_get_cached(void) * size is the one signal the VFS layer offers portably. */ static bool playlist_cached_is_reusable(const playlist_config_t *config) { + playlist_cached_apply_writes(); if (!playlist_cached || playlist_cached_stale) return false; if (!string_is_equal(playlist_cached->config.path, config->path)) diff --git a/retroarch.c b/retroarch.c index 49dc2e1aaf76..502bbd4ee649 100644 --- a/retroarch.c +++ b/retroarch.c @@ -8920,8 +8920,21 @@ static bool retroarch_parse_input_and_config( kind = TASK_DBSCAN_ITEM_REFRESH_ALL; arg = NULL; } + /* Cut to fit the chain's path buffers, a path could + * name a folder above the one asked for. */ + if (arg && strlen(arg) >= PATH_MAX_LENGTH) + { + frontend_driver_attach_console(); + fprintf(stderr, "--%s is too long\n", + c == RA_OPT_SCAN_REFRESH ? "scan-refresh" : "scan"); + exit(1); + } cli_scan = true; + /* The totals and errors are the command's output: the + * config's log_verbosity must not turn them off, which + * on Windows detaches the console. */ verbosity_enable(); + retroarch_override_setting_set(RARCH_OVERRIDE_SETTING_VERBOSITY, NULL); retroarch_override_setting_set(RARCH_OVERRIDE_SETTING_DATABASE_SCAN, NULL); if (!retroarch_cli_scan_add(&scan_items, &scan_count, &scan_cap, kind, arg)) diff --git a/samples/playlist/playlist_parity_test.c b/samples/playlist/playlist_parity_test.c index 156f3b57dae7..f3b6f73908a7 100644 --- a/samples/playlist/playlist_parity_test.c +++ b/samples/playlist/playlist_parity_test.c @@ -57,6 +57,7 @@ #include "../../core_info.h" #include +#include bool core_info_find(const char *core_path, core_info_t **core_info) { @@ -812,6 +813,101 @@ static void check_cache_reuse(const char *path, bool compressed, playlist_free_cached(); } + +/* A task on a worker writes the playlist file the menu has cached. + * The writer leaves a note for the main thread rather than touching + * the cached playlist, which the main thread frees and installs at + * will; the main thread applies the note when it next asks the cache. + * The worker's write keeps the file's size, so the note alone must + * keep the cache from serving what is on disk no longer. A stress + * round follows: the worker rewrites while the main thread reuses and + * frees the cache (under ASan, and TSan with SANITIZER=thread). */ +struct cache_writer +{ + const char *path; + unsigned rounds; +}; + +static void cache_writer_name(const char *path, const char *name) +{ + playlist_config_t config; + playlist_t *pl; + config_defaults(&config, path); + if (!(pl = playlist_init(&config))) + return; + playlist_set_default_core_name(pl, name); + playlist_write_file(pl); + playlist_free(pl); +} + +static void cache_writer_run(void *arg) +{ + struct cache_writer *w = (struct cache_writer*)arg; + unsigned i; + for (i = 0; i < w->rounds; i++) + cache_writer_name(w->path, (i & 1) ? "Core X" : "Core Y"); +} + +static void lane_cache_worker_write(void) +{ + char path[512]; + playlist_config_t config; + struct cache_writer w; + sthread_t *t; + playlist_t *pl; + unsigned had = failures; + unsigned i; + + snprintf(path, sizeof(path), "%s/worker.lpl", fixture_dir); + CHECK(write_whole(path, json_full), "worker lane: fixture write"); + /* the writer's own form, so every later write has its size */ + cache_writer_name(path, "Core Z"); + config_defaults(&config, path); + + playlist_free_cached(); + CHECK(playlist_init_cached(&config), "worker lane: cached init"); + pl = playlist_get_cached(); + CHECK(pl && streq(playlist_get_default_core_name(pl), "Core Z"), + "worker lane: cached playlist"); + + w.path = path; + w.rounds = 1; + CHECK((t = sthread_create(cache_writer_run, &w)) != NULL, + "worker lane: writer thread"); + if (t) + sthread_join(t); + + CHECK(playlist_init_cached(&config), "worker lane: cached init after"); + pl = playlist_get_cached(); + CHECK(pl && streq(playlist_get_default_core_name(pl), "Core Y"), + "worker lane: a same-size write from a worker was served " + "from the stale cache"); + + /* concurrent: the worker writes while the main thread reuses, + * reads and frees */ + w.rounds = 200; + if ((t = sthread_create(cache_writer_run, &w))) + { + for (i = 0; i < 400; i++) + { + if (playlist_init_cached(&config) && (pl = playlist_get_cached())) + CHECK(playlist_get_default_core_name(pl) != NULL, + "worker lane: stress read"); + if (i & 1) + playlist_free_cached(); + } + sthread_join(t); + } + CHECK(playlist_init_cached(&config), "worker lane: final init"); + pl = playlist_get_cached(); + /* the last of 200 writes, round 199, wrote "Core X" */ + CHECK(pl && streq(playlist_get_default_core_name(pl), "Core X"), + "worker lane: the cache missed the worker's last write"); + playlist_free_cached(); + if (failures == had) + fprintf(stderr, "[pass] worker-write cache lane\n"); +} + static void lane_cache_reuse(void) { char path[512]; @@ -1660,6 +1756,7 @@ int main(int argc, char *argv[]) lane_budgeted_autofix(); lane_abort_midway(); lane_cache_reuse(); + lane_cache_worker_write(); lane_switch_playlist_never_shows_previous(); lane_switch_supersedes_pending(); lane_saf_slow_reads(); diff --git a/samples/tasks/database/database_scan_test.c b/samples/tasks/database/database_scan_test.c index 291d7ab3121b..416ec77ad746 100644 --- a/samples/tasks/database/database_scan_test.c +++ b/samples/tasks/database/database_scan_test.c @@ -201,6 +201,8 @@ settings_t *config_get_ptr(void) } static bool invalid_content_pushed = false; +/* How often a refused scan or chain said why. */ +static unsigned busy_pushed = 0; /* Additional stubs for retroarch-core symbols referenced transitively, * to avoid pulling in retroarch.c, runloop.c, frontend drivers, and the @@ -214,6 +216,9 @@ void runloop_msg_queue_push(const char *msg, size_t len, (void)flush; (void)title; (void)icon; (void)category; if (msg && !strcmp(msg, "invalid content")) invalid_content_pushed = true; + if (msg && !strcmp(msg, + msg_hash_to_str(MSG_MANUAL_CONTENT_SCAN_ALREADY_RUNNING))) + busy_pushed++; } /* Whether the scan prints what --scan prints. */ @@ -9164,6 +9169,23 @@ static void chain_done_cb(const task_dbscan_chain_result_t *res, chain_dones++; } +/* A done callback that starts the next chain: the one ending must no + * longer count as running. */ +static const char *chain_restart_arg; +static bool chain_restart_ok; + +static task_dbscan_item_t *chain_items(size_t n, const char *first, + const char *rest); + +static void chain_done_restart_cb(const task_dbscan_chain_result_t *res, + void *user_data) +{ + chain_done_cb(res, user_data); + chain_restart_ok = task_push_dbscan_chain( + chain_items(1, chain_restart_arg, NULL), 1, + chain_item_cb, chain_done_cb, &chain_seen); +} + /* A --scan item for @arg; with @core, an override naming it. */ static void chain_item(task_dbscan_item_t *it, const char *arg, const char *core) @@ -9261,6 +9283,7 @@ static void chain_lane(const char *root, const char *pl_dir, task_dbscan_item_t *items; bool pushed, refused, captured, back, running; bool scan_refused; + unsigned busy_before, busy_scan; sprintf(a, "%s/chain_a", root); sprintf(b, "%s/chain_b", root); @@ -9364,11 +9387,17 @@ static void chain_lane(const char *root, const char *pl_dir, && chain_saw(0, slow, TASK_DBSCAN_RAN, chain_added[0]) && chain_totals.ran == 1 && chain_totals.cancelled, what, detail); + /* what the menu holds before the chain, to find again after it */ + manual_content_scan_reset_settings(); + strlcpy(manual_content_scan_get_system_name_custom_ptr(), "Kept Name", + manual_content_scan_get_system_name_custom_size()); pushed = chain_push(chain_items(1, a, NULL), 1); running = task_dbscan_chain_is_running(); refused = !task_push_dbscan_chain(chain_items(1, b, NULL), 1, chain_item_cb, chain_done_cb, &chain_seen); + busy_before = busy_pushed; scan_refused = !task_push_dbscan(b, scan_cb, NULL); + busy_scan = busy_pushed - busy_before; chain_wait(false); sprintf(what, "chain %s: a second chain refused", q); sprintf(detail, "%u items, done %u", chain_seen, chain_dones); @@ -9376,6 +9405,39 @@ static void chain_lane(const char *root, const char *pl_dir, && chain_saw(0, a, TASK_DBSCAN_RAN, 0), what, detail); sprintf(what, "chain %s: running from the push to the end", q); check(running && !task_dbscan_chain_is_running(), what, ""); + sprintf(what, "chain %s: a scan refused while it runs says why", q); + sprintf(detail, "refused %d, told %u", (int)scan_refused, busy_scan); + check(scan_refused && busy_scan == 1, what, detail); + sprintf(what, "chain %s: the Manual Scan settings come back", q); + check(!strcmp(manual_content_scan_get_system_name_custom_ptr(), + "Kept Name"), what, + manual_content_scan_get_system_name_custom_ptr()); + manual_content_scan_reset_settings(); + + /* the done callback starts the next chain */ + chain_seen = 0; + chain_dones = 0; + chain_ud_ok = true; + chain_restart_ok = false; + chain_restart_arg = b; + pushed = task_push_dbscan_chain(chain_items(1, a, NULL), 1, + chain_item_cb, chain_done_restart_cb, &chain_seen); + { + time_t started = time(NULL); + while ( pushed && chain_dones < 2 + && difftime(time(NULL), started) <= SCAN_TIMEOUT_SECONDS) + { + task_queue_check(); + retro_sleep(1); + } + } + sprintf(what, "chain %s: its done callback may start another", q); + sprintf(detail, "pushed %d, restarted %d, done %u, items %u", + (int)pushed, (int)chain_restart_ok, chain_dones, chain_seen); + check( pushed && chain_restart_ok && chain_dones == 2 + && chain_saw(0, a, TASK_DBSCAN_RAN, 0) + && chain_saw(1, b, TASK_DBSCAN_RAN, chain_added[1]), + what, detail); /* Held, so the scan cannot end before the chain is pushed */ slow_reset(sz_a, 0); diff --git a/tasks/task_database.c b/tasks/task_database.c index 5a68ef32bd16..697da3aba3a8 100644 --- a/tasks/task_database.c +++ b/tasks/task_database.c @@ -4214,8 +4214,7 @@ static bool manual_scan_end_flush_tick( bool task_push_dbscan(const char *fullpath, retro_task_callback_t cb, void *user_data) { - /* It would run with the settings the chain last loaded. */ - if (task_dbscan_chain_is_running()) + if (task_dbscan_chain_refuse()) return false; manual_content_scan_set_menu_content_dir(fullpath); return task_push_manual_content_scan(false, cb, user_data); @@ -5927,6 +5926,8 @@ typedef struct size_t next; size_t all_next; task_dbscan_chain_result_t totals; + /* the Manual Scan settings the chain found, put back when it ends */ + manual_content_scan_saved_t *saved; bool all_running; bool all_counted; /* a playlist of the refresh-all had settings */ bool cli_output; @@ -5950,8 +5951,14 @@ static void dbscan_chain_free_items(task_dbscan_item_t *items, free(items); } +/* Ends the chain: what it found in the Manual Scan settings goes back, + * and it is no longer running when @done_cb hears the totals, so that + * callback may start another. */ static void dbscan_chain_finish(dbscan_chain_t *chain) { + manual_content_scan_restore_settings(chain->saved); + chain->saved = NULL; + dbscan_chain_running = NULL; if (chain->done_cb) chain->done_cb(&chain->totals, chain->user_data); dbscan_chain_free_items(chain->items, chain->count); @@ -5961,7 +5968,6 @@ static void dbscan_chain_finish(dbscan_chain_t *chain) string_list_free(chain->playlists); free(chain->name); free(chain); - dbscan_chain_running = NULL; } static void dbscan_chain_fold(dbscan_chain_t *chain, const char *name, @@ -6554,8 +6560,7 @@ static void dbscan_prep_free(retro_task_t *task) /* Pushes a prep task for @prep; false, @prep freed, if it cannot. */ static bool dbscan_chain_push_prep(dbscan_chain_t *chain, - dbscan_prep_t *prep, retro_task_handler_t handler, - retro_task_callback_t cb) + dbscan_prep_t *prep, bool load, retro_task_callback_t cb) { retro_task_t *task = prep ? task_init() : NULL; @@ -6569,8 +6574,12 @@ static bool dbscan_chain_push_prep(dbscan_chain_t *chain, } return false; } - /* Silent: no title, no notifications. */ - task->handler = handler; + /* Silent: no title, no notifications. Each handler by name, so the + * thread-read audit takes both as worker entries. */ + if (load) + task->handler = dbscan_load_handler; + else + task->handler = dbscan_prep_handler; task->state = prep; task->user_data = chain; task->callback = cb; @@ -6616,7 +6625,7 @@ static bool dbscan_chain_push_load(dbscan_chain_t *chain, ? settings->paths.directory_menu_content : NULL); prep->kind = DBSCAN_PREP_LOAD; } - if (dbscan_chain_push_prep(chain, prep, dbscan_load_handler, + if (dbscan_chain_push_prep(chain, prep, true, dbscan_load_cb)) return true; fill_pathname(name, path_basename(path), "", sizeof(name)); @@ -6723,7 +6732,7 @@ static bool dbscan_chain_push_list(dbscan_chain_t *chain) prep = NULL; } } - return dbscan_chain_push_prep(chain, prep, dbscan_prep_handler, + return dbscan_chain_push_prep(chain, prep, false, dbscan_prep_cb); } @@ -6786,6 +6795,29 @@ static bool dbscan_chain_scan_finder(retro_task_t *task, void *user_data) && !(task_get_flags(task) & RETRO_TASK_FLG_FINISHED); } +static void dbscan_chain_refusal(void) +{ + const char *_msg = msg_hash_to_str(MSG_MANUAL_CONTENT_SCAN_ALREADY_RUNNING); + runloop_msg_queue_push(_msg, strlen(_msg), 1, 150, true, NULL, + MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_ERROR); +} + +bool task_dbscan_chain_refuse(void) +{ + if (!dbscan_chain_running) + return false; + dbscan_chain_refusal(); + return true; +} + +bool task_dbscan_chain_refuse_start(void) +{ + if (task_dbscan_chain_can_start()) + return false; + dbscan_chain_refusal(); + return true; +} + bool task_dbscan_chain_can_start(void) { task_finder_data_t find_data; @@ -6823,9 +6855,11 @@ bool task_push_dbscan_chain(task_dbscan_item_t *items, size_t count, chain->user_data = user_data; chain->cli_output = retroarch_override_setting_is_set( RARCH_OVERRIDE_SETTING_DATABASE_SCAN, NULL); + if (!(chain->saved = manual_content_scan_save_settings())) + goto error; dbscan_chain_running = chain; - if (dbscan_chain_push_prep(chain, prep, dbscan_prep_handler, + if (dbscan_chain_push_prep(chain, prep, false, dbscan_prep_cb)) return true; dbscan_chain_running = NULL; @@ -6835,6 +6869,8 @@ bool task_push_dbscan_chain(task_dbscan_item_t *items, size_t count, if (prep) free(prep->dir); free(prep); + if (chain) + free(chain->saved); free(chain); dbscan_chain_free_items(items, count); return false; diff --git a/tasks/tasks_internal.h b/tasks/tasks_internal.h index 9f920c1196c4..805da72e559a 100644 --- a/tasks/tasks_internal.h +++ b/tasks/tasks_internal.h @@ -411,6 +411,11 @@ bool task_dbscan_chain_is_running(void); /* False while task_push_dbscan_chain() would refuse: a chain or a * manual scan runs. Main thread only. */ bool task_dbscan_chain_can_start(void); +/* A scan asked for while a chain runs would use the settings the chain + * last loaded: true, with the message shown, when it must be refused. */ +bool task_dbscan_chain_refuse(void); +/* The same for a new chain, which also waits for a running scan. */ +bool task_dbscan_chain_refuse_start(void); bool task_push_manual_content_scan( bool do_menu_refresh, diff --git a/tools/thread_read_audit.py b/tools/thread_read_audit.py index 6b6ba2a179b5..92c289e0dfe7 100755 --- a/tools/thread_read_audit.py +++ b/tools/thread_read_audit.py @@ -292,7 +292,7 @@ def enclosing_function(text, pos): return text[start if start >= 0 else 0:end if end >= 0 else len(text)] -def source_entries(root, main_thread=None): +def source_entries(root, main_thread=None, forwarded=None): entries = {} # handler name -> [path, every assignment flagged main-thread] handlers = {} @@ -320,6 +320,14 @@ def source_entries(root, main_thread=None): entries.setdefault(m.group(1), rel) clears = MAIN_CLEAR_RE.search(text) is not None for m in HANDLER_RE.finditer(text): + # A handler set from a retro_task_handler_t parameter names + # no function, and the handlers its callers pass are never + # entries: the audit cannot see them, so it says so. + if re.search(r"\bretro_task_handler_t\s+%s\b" + % re.escape(m.group(1)), text): + if forwarded is not None: + forwarded.append((rel, m.group(1))) + continue main = (not clears and MAIN_SET_RE.search( enclosing_function(text, m.start())) is not None) site = handlers.setdefault(m.group(1), [rel, True]) @@ -417,7 +425,8 @@ def load_allow(path): def run(binary, root, allow_path, list_unaudited=False, objdump="objdump"): main_thread = {} - entries = source_entries(root, main_thread) + forwarded = [] + entries = source_entries(root, main_thread, forwarded) try: out = subprocess.run([objdump, "-d", binary], capture_output=True, text=True, check=True) @@ -436,6 +445,12 @@ def run(binary, root, allow_path, list_unaudited=False, print("tools/thread_read_allow.list.") for entry, fn, reader in findings: print("%s -> %s calls %s" % (entry, fn, reader)) + if forwarded: + print("%d task handler(s) set from a parameter, so the handlers" + % len(forwarded)) + print("passed in are not audited. Assign each by name:") + for rel, name in forwarded: + print(" %s: task->handler = %s" % (rel, name)) if list_unaudited: absent = sorted(e for e in entries if e not in defined) print("%d source entr%s absent from this binary - compiled out" @@ -452,7 +467,7 @@ def run(binary, root, allow_path, list_unaudited=False, len(boundaries), "y" if len(boundaries) == 1 else "ies", len(main_thread)), file=sys.stderr) - return 1 if findings else 0 + return 1 if findings or forwarded else 0 FIXTURE = """ @@ -520,11 +535,22 @@ def selftest(): " task_set_flags(task, RETRO_TASK_FLG_MAIN_THREAD," " false);\n" "}\n") + # A helper that sets whatever handler it is given hides it. + with open(os.path.join(td, "fwd.c"), "w") as f: + f.write("static void push_any(retro_task_t *task,\n" + " retro_task_handler_t handler)\n" + "{\n" + " task->handler = handler;\n" + "}\n") if subprocess.run(["gcc", "-O0", src, "-o", binp, "-lpthread"]).returncode: print("selftest: fixture build failed") return 1 - entries = source_entries(td) + forwarded = [] + entries = source_entries(td, None, forwarded) + if forwarded != [("fwd.c", "handler")] or "handler" in entries: + print("selftest: FAIL forwarded handler %r" % forwarded) + return 1 out = subprocess.run(["objdump", "-d", binp], capture_output=True, text=True, check=True) calls, defined = call_graph(out.stdout.splitlines()) diff --git a/ui/companion/companion_core.c b/ui/companion/companion_core.c index db02845d7005..e29fb41fd80f 100644 --- a/ui/companion/companion_core.c +++ b/ui/companion/companion_core.c @@ -1849,6 +1849,15 @@ static void companion_core_scan_finished(retro_task_t *task, } #endif +const char *companion_core_scan_refusal(void) +{ +#ifdef HAVE_LIBRETRODB + if (task_dbscan_chain_is_running()) + return msg_hash_to_str(MSG_MANUAL_CONTENT_SCAN_ALREADY_RUNNING); +#endif + return "Scanning is not available in this build."; +} + bool companion_core_request_scan(companion_core_t *core, const char *path) { #ifdef HAVE_LIBRETRODB diff --git a/ui/companion/companion_core.h b/ui/companion/companion_core.h index 4965a4deb080..e3d3a00e9c23 100644 --- a/ui/companion/companion_core.h +++ b/ui/companion/companion_core.h @@ -272,6 +272,8 @@ void companion_core_event_command(companion_core_t *core, * list has been reset. Returns false when no scan could be started * (no libretrodb in this build, or the task could not be pushed). */ bool companion_core_request_scan(companion_core_t *core, const char *path); +/* Why companion_core_request_scan() refused, for the status line. */ +const char *companion_core_scan_refusal(void); /* --- Companion settings (retroarch.cfg) ------------------------------- */ diff --git a/ui/companion/test/companion_core_stubs.c b/ui/companion/test/companion_core_stubs.c index 3c62adaf6e68..9a052c797435 100644 --- a/ui/companion/test/companion_core_stubs.c +++ b/ui/companion/test/companion_core_stubs.c @@ -219,6 +219,8 @@ bool task_push_dbscan(const char *path, retro_task_callback_t cb, return true; } +bool task_dbscan_chain_is_running(void) { return false; } + /* Entries and macros (input_driver.c): kept by number as their text. An * entry reads when it has a colon, a macro when it has a digit, as a * stand-in for the real parsers; sixteen of each at most. */ diff --git a/ui/drivers/ui_cocoa_companion.m b/ui/drivers/ui_cocoa_companion.m index d30c788eacd3..3b9cb100365e 100644 --- a/ui/drivers/ui_cocoa_companion.m +++ b/ui/drivers/ui_cocoa_companion.m @@ -4227,7 +4227,7 @@ - (void)scanDirectory:(id)sender if (companion_core_request_scan(wimp->core, [path UTF8String])) [self setStatus:"Scanning..."]; else - [self setStatus:"Scanning is not available in this build."]; + [self setStatus:companion_core_scan_refusal()]; } /* The companion's own Load Core window (like Qt's LoadCoreWindow): diff --git a/ui/drivers/ui_win32_companion.c b/ui/drivers/ui_win32_companion.c index 2834b8308687..b05b7181d2c6 100644 --- a/ui/drivers/ui_win32_companion.c +++ b/ui/drivers/ui_win32_companion.c @@ -4606,7 +4606,7 @@ static void cw_scan_directory(ui_companion_win32_wimp_t *w) if (companion_core_request_scan(w->core, dir)) cw_status_set(w, "Scanning..."); else - cw_status_set(w, "Scanning is not available in this build."); + cw_status_set(w, companion_core_scan_refusal()); } static void cw_context_menu(ui_companion_win32_wimp_t *w, HWND from, From e7a1c610cd32d7f928f2a172cfde36eff99ae257 Mon Sep 17 00:00:00 2001 From: LibretroAdmin Date: Wed, 4 Nov 2026 18:44:13 +0100 Subject: [PATCH 19/21] playlist: header-only parse for the scan chain's prep The prep task needs a playlist's saved scan settings and nothing else, yet parsed every entry and, through autofix, could write the file back from a worker. playlist_parse_begin_header() stops at "items", rebases the scan record in memory when the base content directory moved, and marks the playlist so playlist_write_file() refuses it. Lane: playlist_parity_test header_only (entries skipped, scan dir rebased, file bytes untouched, old format). --- playlist.c | 126 +++++++++++++++----- playlist.h | 6 + samples/playlist/playlist_parity_test.c | 98 +++++++++++++++ samples/tasks/database/Makefile | 2 +- samples/tasks/database/database_scan_test.c | 9 +- tasks/task_database.c | 8 +- 6 files changed, 210 insertions(+), 39 deletions(-) diff --git a/playlist.c b/playlist.c index 2931e5b5286e..4ee2797da8b6 100644 --- a/playlist.c +++ b/playlist.c @@ -77,7 +77,9 @@ enum content_playlist_flags CNT_PLAYLIST_FLG_MOD = (1 << 0), CNT_PLAYLIST_FLG_OLD_FMT = (1 << 1), CNT_PLAYLIST_FLG_COMPRESSED = (1 << 2), - CNT_PLAYLIST_FLG_CACHED_EXT = (1 << 3) + CNT_PLAYLIST_FLG_CACHED_EXT = (1 << 3), + /* read by playlist_parse_begin_header(): no entries, never written */ + CNT_PLAYLIST_FLG_HEADER_ONLY = (1 << 4) }; struct content_playlist @@ -113,7 +115,9 @@ enum json_ctx_flags JSON_CTX_FLG_IN_ITEMS = (1 << 0), JSON_CTX_FLG_IN_SUBSYSTEM_CONTENT = (1 << 1), JSON_CTX_FLG_CAPACITY_EXCEEDED = (1 << 2), - JSON_CTX_FLG_OOM = (1 << 3) + JSON_CTX_FLG_OOM = (1 << 3), + JSON_CTX_FLG_HEADER_ONLY = (1 << 4), /* stop at "items" */ + JSON_CTX_FLG_HEADER_DONE = (1 << 5) }; /* The runtime and last-played values live packed inside the entry and @@ -2278,6 +2282,8 @@ void playlist_write_file(playlist_t *playlist) if ( !playlist || !*playlist->config.path + /* a header alone would replace the file's entries with none */ + || (playlist->flags & CNT_PLAYLIST_FLG_HEADER_ONLY) || !( (playlist->flags & CNT_PLAYLIST_FLG_MOD) #if defined(HAVE_COMPRESSION) || (pl_compressed != playlist->config.compress) @@ -3165,7 +3171,13 @@ static bool JSONObjectMemberHandler(void *context, const char *pValue, size_t le break; case 'i': if (!strcmp(pValue, "items")) - pCtx->flags |= JSON_CTX_FLG_IN_ITEMS; + { + /* the writer puts every header field before the items */ + if (pCtx->flags & JSON_CTX_FLG_HEADER_ONLY) + pCtx->flags |= JSON_CTX_FLG_HEADER_DONE; + else + pCtx->flags |= JSON_CTX_FLG_IN_ITEMS; + } break; case 'l': if (!strcmp(pValue, "label_display_mode")) @@ -3267,6 +3279,7 @@ struct playlist_parse enum playlist_parse_phase phase; bool res; bool autofix_scan_done; + bool header_only; }; static void playlist_parse_close_io(playlist_parse_t *p) @@ -3305,6 +3318,35 @@ static void playlist_parse_close_io(playlist_parse_t *p) /* Decide what follows the parse: the autofix pass when it applies, */ /* completion otherwise. Mirrors the condition playlist_init */ /* historically evaluated after playlist_read_file returned. */ +/* The scan record's paths under the base content directory @p's + * config names, as autofix moves them. */ +static void playlist_rebase_scan_record(playlist_t *playlist, + size_t oldref_len, size_t newref_len) +{ + char tmp[PATH_MAX_LENGTH]; + + if (playlist->scan_record.content_dir && *playlist->scan_record.content_dir) + { + tmp[0] = '\0'; + path_replace_base_path_and_convert_to_local_file_system(tmp, + playlist->scan_record.content_dir, + playlist->base_content_directory, oldref_len, + playlist->config.base_content_directory, newref_len, + sizeof(tmp)); + playlist_entry_replace_str(&playlist->scan_record.content_dir, tmp); + } + if (playlist->scan_record.dat_file_path && *playlist->scan_record.dat_file_path) + { + tmp[0] = '\0'; + path_replace_base_path_and_convert_to_local_file_system(tmp, + playlist->scan_record.dat_file_path, + playlist->base_content_directory, oldref_len, + playlist->config.base_content_directory, newref_len, + sizeof(tmp)); + playlist_entry_replace_str(&playlist->scan_record.dat_file_path, tmp); + } +} + static void playlist_parse_enter_autofix(playlist_parse_t *p) { playlist_t *playlist = p->playlist; @@ -3317,6 +3359,23 @@ static void playlist_parse_enter_autofix(playlist_parse_t *p) return; } + /* The header alone: its scan record moves as autofix would move + * it, in memory - there are no entries to fix and nothing is + * saved. */ + if (p->header_only) + { + if ( playlist->config.autofix_paths + && !string_is_equal(playlist->base_content_directory, + playlist->config.base_content_directory) + && playlist->base_content_directory + && *playlist->base_content_directory) + playlist_rebase_scan_record(playlist, + strlen(playlist->base_content_directory), + strlen(playlist->config.base_content_directory)); + p->phase = PLAYLIST_PARSE_PHASE_DONE; + return; + } + if ( playlist->config.autofix_paths && !string_is_equal(playlist->base_content_directory, playlist->config.base_content_directory) @@ -3421,6 +3480,11 @@ static int playlist_parse_step_json(playlist_parse_t *p, stop = t; goto stopped; } + if (p->context.flags & JSON_CTX_FLG_HEADER_DONE) + { + stop = RJSON_DONE; + goto stopped; + } } stopped: @@ -3730,35 +3794,8 @@ static int playlist_parse_step_autofix(playlist_parse_t *p, { p->autofix_scan_done = true; - /* Fix scan record content directory */ - if (playlist->scan_record.content_dir && *playlist->scan_record.content_dir) - { - tmp_entry_path[0] = '\0'; - path_replace_base_path_and_convert_to_local_file_system( - tmp_entry_path, - playlist->scan_record.content_dir, - playlist->base_content_directory, p->oldref_len, - playlist->config.base_content_directory, p->newref_len, - sizeof(tmp_entry_path)); - - playlist_entry_replace_str(&playlist->scan_record.content_dir, - tmp_entry_path); - } - - /* Fix scan record arcade DAT file */ - if (playlist->scan_record.dat_file_path && *playlist->scan_record.dat_file_path) - { - tmp_entry_path[0] = '\0'; - path_replace_base_path_and_convert_to_local_file_system( - tmp_entry_path, - playlist->scan_record.dat_file_path, - playlist->base_content_directory, p->oldref_len, - playlist->config.base_content_directory, p->newref_len, - sizeof(tmp_entry_path)); - - playlist_entry_replace_str(&playlist->scan_record.dat_file_path, - tmp_entry_path); - } + /* Fix the scan record's content directory and DAT file */ + playlist_rebase_scan_record(playlist, p->oldref_len, p->newref_len); } } @@ -3781,7 +3818,8 @@ static int playlist_parse_step_autofix(playlist_parse_t *p, return 1; } -playlist_parse_t *playlist_parse_begin(const playlist_config_t *config) +static playlist_parse_t *playlist_parse_begin_ex( + const playlist_config_t *config, bool header_only) { playlist_parse_t *p = NULL; playlist_t *playlist = (playlist_t*)malloc(sizeof(*playlist)); @@ -3822,6 +3860,11 @@ playlist_parse_t *playlist_parse_begin(const playlist_config_t *config) p->playlist = playlist; p->res = true; p->phase = PLAYLIST_PARSE_PHASE_DONE; /* until proven otherwise */ + if ((p->header_only = header_only)) + { + playlist->flags |= CNT_PLAYLIST_FLG_HEADER_ONLY; + p->context.flags |= JSON_CTX_FLG_HEADER_ONLY; + } #if defined(HAVE_COMPRESSION) /* Always use RZIP interface when reading playlists @@ -3912,6 +3955,13 @@ playlist_parse_t *playlist_parse_begin(const playlist_config_t *config) return p; } + /* The old format has no scan settings: a header read is done. */ + if (p->header_only) + { + playlist_parse_enter_autofix(p); + return p; + } + /* Heap-held: at six entries this is 3 KiB even at the console * path length, on the playlist load path that runs from task * threads. On allocation failure the legacy-format file simply @@ -3931,6 +3981,16 @@ playlist_parse_t *playlist_parse_begin(const playlist_config_t *config) return NULL; } +playlist_parse_t *playlist_parse_begin(const playlist_config_t *config) +{ + return playlist_parse_begin_ex(config, false); +} + +playlist_parse_t *playlist_parse_begin_header(const playlist_config_t *config) +{ + return playlist_parse_begin_ex(config, true); +} + int playlist_parse_step(playlist_parse_t *p, bool (*budget_cb)(void *), void *budget_ud) { diff --git a/playlist.h b/playlist.h index 7ccc98d26192..0e21c6377b4b 100644 --- a/playlist.h +++ b/playlist.h @@ -259,6 +259,12 @@ playlist_t *playlist_init(const playlist_config_t *config); typedef struct playlist_parse playlist_parse_t; playlist_parse_t *playlist_parse_begin(const playlist_config_t *config); +/* The same, for what comes ahead of the entries only - the defaults + * and the scan record, which the writer puts before them - as a + * playlist with no entries that playlist_write_file() never writes. + * The scan record's paths move to the configured base content + * directory as a full read's autofix would move them, in memory. */ +playlist_parse_t *playlist_parse_begin_header(const playlist_config_t *config); int playlist_parse_step(playlist_parse_t *p, bool (*budget_cb)(void *), void *budget_ud); playlist_t *playlist_parse_end(playlist_parse_t *p); diff --git a/samples/playlist/playlist_parity_test.c b/samples/playlist/playlist_parity_test.c index f3b6f73908a7..23751547f401 100644 --- a/samples/playlist/playlist_parity_test.c +++ b/samples/playlist/playlist_parity_test.c @@ -908,6 +908,103 @@ static void lane_cache_worker_write(void) fprintf(stderr, "[pass] worker-write cache lane\n"); } +/* The whole file, for comparing it before and after; caller frees. */ +static char *slurp(const char *path, long *len) +{ + char *buf = NULL; + FILE *f = fopen(path, "rb"); + *len = -1; + if (!f) + return NULL; + if ( !fseek(f, 0, SEEK_END) + && (*len = ftell(f)) >= 0 + && !fseek(f, 0, SEEK_SET) + && (buf = (char*)malloc((size_t)*len + 1))) + { + if (fread(buf, 1, (size_t)*len, f) != (size_t)*len) + { + free(buf); + buf = NULL; + } + } + fclose(f); + return buf; +} + +static playlist_t *header_read(const playlist_config_t *config) +{ + playlist_parse_t *p = playlist_parse_begin_header(config); + if (!p) + return NULL; + if (playlist_parse_step(p, NULL, NULL) < 0) + { + playlist_parse_abort(p); + return NULL; + } + return playlist_parse_end(p); +} + +/* playlist_parse_begin_header(): what comes ahead of the entries and + * nothing after; under a moved base content directory the scan record + * moves in memory while the file stays as it was; the result is never + * written; the old format, which has no scan settings, reads empty. */ +static void lane_header_only(void) +{ + char path[512]; + playlist_config_t config; + playlist_t *pl; + char *before, *after; + long blen, alen; + unsigned had = failures; + + snprintf(path, sizeof(path), "%s/header.lpl", fixture_dir); + CHECK(write_whole(path, json_full), "header lane: fixture write"); + config_defaults(&config, path); + + pl = header_read(&config); + CHECK(pl != NULL, "header lane: read"); + if (pl) + { + CHECK(playlist_size(pl) == 0, "header lane: %u entries read", + (unsigned)playlist_size(pl)); + CHECK(streq(playlist_get_default_core_name(pl), "Core A"), + "header lane: default core name"); + CHECK(streq(playlist_get_scan_content_dir(pl), "/games/snes"), + "header lane: scan content dir"); + playlist_free(pl); + } + + /* a base moved since the save: autofix's move, in memory only */ + before = slurp(path, &blen); + playlist_config_set_base_content_directory(&config, "/moved"); + pl = header_read(&config); + CHECK(pl && streq(playlist_get_scan_content_dir(pl), "/moved/snes"), + "header lane: scan content dir under the moved base"); + if (pl) + { + /* and never written, even when asked */ + playlist_set_default_core_name(pl, "Core Changed"); + playlist_write_file(pl); + playlist_free(pl); + } + after = slurp(path, &alen); + CHECK(before && after && blen == alen && !memcmp(before, after, (size_t)blen), + "header lane: the file changed under a header read"); + free(before); + free(after); + + CHECK(write_whole(path, old_format), "header lane: old format write"); + config_defaults(&config, path); + pl = header_read(&config); + CHECK(pl && playlist_size(pl) == 0 + && !playlist_get_scan_content_dir(pl), + "header lane: the old format reads empty"); + playlist_free(pl); + + if (failures == had) + fprintf(stderr, "[pass] header-only lane\n"); +} + static void lane_cache_reuse(void) { char path[512]; @@ -1757,6 +1854,7 @@ int main(int argc, char *argv[]) lane_abort_midway(); lane_cache_reuse(); lane_cache_worker_write(); + lane_header_only(); lane_switch_playlist_never_shows_previous(); lane_switch_supersedes_pending(); lane_saf_slow_reads(); diff --git a/samples/tasks/database/Makefile b/samples/tasks/database/Makefile index b90d13ce9b8e..23d3b140fa72 100644 --- a/samples/tasks/database/Makefile +++ b/samples/tasks/database/Makefile @@ -268,7 +268,7 @@ SCAN_TEST_WRAPS := -Wl,--wrap=libretrodb_open -Wl,--wrap=task_queue_park \ -Wl,--wrap=task_nbio_slice_within_budget \ -Wl,--wrap=fill_pathname_join_delim \ -Wl,--wrap=dir_list_iter_new -Wl,--wrap=dir_list_iter_step \ - -Wl,--wrap=playlist_parse_begin -Wl,--wrap=task_init + -Wl,--wrap=playlist_parse_begin_header -Wl,--wrap=task_init ifeq ($(HAVE_THREADS), 1) SCAN_TEST_WRAPS += -Wl,--wrap=sthread_create endif diff --git a/samples/tasks/database/database_scan_test.c b/samples/tasks/database/database_scan_test.c index 416ec77ad746..3058792e9204 100644 --- a/samples/tasks/database/database_scan_test.c +++ b/samples/tasks/database/database_scan_test.c @@ -9539,12 +9539,15 @@ int __wrap_dir_list_iter_step(dir_list_iter_t *iter, return __real_dir_list_iter_step(iter, within_budget, ud); } -playlist_parse_t *__real_playlist_parse_begin(const playlist_config_t *config); -playlist_parse_t *__wrap_playlist_parse_begin(const playlist_config_t *config) +/* Linked with --wrap: a refresh's load reads the playlist's header. */ +playlist_parse_t *__real_playlist_parse_begin_header( + const playlist_config_t *config); +playlist_parse_t *__wrap_playlist_parse_begin_header( + const playlist_config_t *config) { refresh_began_one(REFRESH_LOAD); prep_cancel_begun(); - return __real_playlist_parse_begin(config); + return __real_playlist_parse_begin_header(config); } /* Linked with --wrap: the task_init() this counts down to fails, as diff --git a/tasks/task_database.c b/tasks/task_database.c index 697da3aba3a8..4312c2300677 100644 --- a/tasks/task_database.c +++ b/tasks/task_database.c @@ -6425,7 +6425,9 @@ static void dbscan_load_handler(retro_task_t *task) if (!path_is_valid(prep->config.path)) prep->load = DBSCAN_LOAD_NO_PLAYLIST; else - prep->parse = playlist_parse_begin(&prep->config); + /* The header alone: the saved settings, never the entries, + * and nothing written. */ + prep->parse = playlist_parse_begin_header(&prep->config); } /* One window per call on the main thread. */ if (prep->iter) @@ -6437,7 +6439,9 @@ static void dbscan_load_handler(retro_task_t *task) if (!path_is_valid(prep->config.path)) prep->load = DBSCAN_LOAD_NO_PLAYLIST; else - prep->parse = playlist_parse_begin(&prep->config); + /* The header alone: the saved settings, never the entries, + * and nothing written. */ + prep->parse = playlist_parse_begin_header(&prep->config); } if (!task_queue_is_threaded()) return; From 659810b47b0e4f59edd935d0b5558af931f8731b Mon Sep 17 00:00:00 2001 From: LibretroAdmin Date: Wed, 4 Nov 2026 18:52:09 +0100 Subject: [PATCH 20/21] scan chain: ask the file system nothing on the main thread The chain's steps ran on the main thread between its tasks and stat'd and realpath'd there: a --scan path's resolve and stat, a --scan-refresh path's two realpaths, the content folder's stat, the saved and override DATs' stat and size, an override core path's realpath. On a network share each of those can stall a frame. - manual_content_scan_fs_read() reads them on any thread, touching nothing but its arguments; the _fs variants of the content-folder, playlist and override setters take its answers instead of asking. The plain setters pass NULL and behave as before, so the menu is unchanged. - A --scan path gets its own prep task (dbscan_path_handler); a --scan-refresh resolves its path inside the load task, and the load reads the folder, the saved DAT and the overrides' paths once the header is parsed. The worker gets copies of the item's strings. - The prep's state is freed in one place. Lane: database_scan_test "fs off main": six items that each stop before their scan, on a threaded queue, with path_stat, path_is_valid, path_is_directory, path_get_size and path_resolve_realpath wrapped: no call on the main thread. Not moved: the scan task's own push still stats the content folder and the DAT (manual_content_scan_get_task_config), shared with the menu. --- manual_content_scan.c | 241 +++++++++-- manual_content_scan.h | 42 ++ samples/tasks/database/Makefile | 10 +- samples/tasks/database/database_scan_test.c | 144 +++++++ tasks/task_database.c | 448 +++++++++++++------- 5 files changed, 684 insertions(+), 201 deletions(-) diff --git a/manual_content_scan.c b/manual_content_scan.c index 094a40009aef..cbcc1c1d63c0 100644 --- a/manual_content_scan.c +++ b/manual_content_scan.c @@ -270,24 +270,30 @@ void manual_content_scan_scrub_file_exts_custom(void) * in, structs out - so both the extension whitelist * and the file system checks live here, with the scan * configuration that owns the path. */ -bool manual_content_scan_dat_path_is_valid( - const char *path, uint64_t *file_size) +/* The extensions a Logiqx XML DAT may legitimately carry: + * .dat and .xml */ +static bool manual_content_scan_dat_ext_is_valid(const char *path) { const char *file_ext = NULL; - int64_t file_size_int; if (!path || !*path) return false; - /* Check file extension (.dat / .xml, the extensions - * a Logiqx XML DAT may legitimately carry) */ file_ext = path_get_extension(path); if (!file_ext || !*file_ext) return false; - if ( !string_is_equal_noncase(file_ext, "dat") - && !string_is_equal_noncase(file_ext, "xml")) + return string_is_equal_noncase(file_ext, "dat") + || string_is_equal_noncase(file_ext, "xml"); +} + +bool manual_content_scan_dat_path_is_valid( + const char *path, uint64_t *file_size) +{ + int64_t file_size_int; + + if (!manual_content_scan_dat_ext_is_valid(path)) return false; /* Ensure file exists */ @@ -307,9 +313,11 @@ bool manual_content_scan_dat_path_is_valid( } /* Checks 'dat_file_path' string and resets it - * if invalid */ -enum manual_content_scan_dat_file_path_status - manual_content_scan_validate_dat_file_path(void) + * if invalid. @known_size, when not NULL, is the file's + * size as read already (<= 0: not there), so the file + * system is not asked again. */ +static enum manual_content_scan_dat_file_path_status + manual_content_scan_validate_dat(const int64_t *known_size) { enum manual_content_scan_dat_file_path_status dat_file_path_status = MANUAL_CONTENT_SCAN_DAT_FILE_UNSET; @@ -317,10 +325,22 @@ enum manual_content_scan_dat_file_path_status /* Check if 'dat_file_path' has been set */ if (*scan_dat_file_path) { - uint64_t file_size; + uint64_t file_size = 0; + bool valid; /* Check if path itself is valid */ - if (manual_content_scan_dat_path_is_valid(scan_dat_file_path, &file_size)) + if (known_size) + { + valid = manual_content_scan_dat_ext_is_valid(scan_dat_file_path) + && *known_size > 0; + if (valid) + file_size = (uint64_t)*known_size; + } + else + valid = manual_content_scan_dat_path_is_valid(scan_dat_file_path, + &file_size); + + if (valid) { uint64_t free_memory = mem_stats_free(); dat_file_path_status = MANUAL_CONTENT_SCAN_DAT_FILE_OK; @@ -361,6 +381,12 @@ enum manual_content_scan_dat_file_path_status return dat_file_path_status; } +enum manual_content_scan_dat_file_path_status + manual_content_scan_validate_dat_file_path(void) +{ + return manual_content_scan_validate_dat(NULL); +} + /* Menu setters */ /* Sets every setting back to its default */ @@ -434,6 +460,12 @@ void manual_content_scan_restore_settings(manual_content_scan_saved_t *sv) * operation. * Returns true if content directory is valid. */ bool manual_content_scan_set_menu_content_dir(const char *content_dir) +{ + return manual_content_scan_set_menu_content_dir_fs(content_dir, NULL); +} + +bool manual_content_scan_set_menu_content_dir_fs(const char *content_dir, + const manual_content_scan_fs_t *fs) { size_t _len; int path_stats; @@ -495,7 +527,7 @@ bool manual_content_scan_set_menu_content_dir(const char *content_dir) /* A file takes the name of the folder that holds it, * as a scan of that folder would */ - path_stats = path_stat(scan_content_dir); + path_stats = fs ? fs->content_stat : path_stat(scan_content_dir); if ( (path_stats & RETRO_VFS_STAT_IS_VALID) && !(path_stats & RETRO_VFS_STAT_IS_DIRECTORY) && fill_pathname_parent_dir_name(scan_system_name_content_dir, @@ -756,6 +788,15 @@ bool manual_content_scan_set_menu_core_name( enum manual_content_scan_playlist_refresh_status manual_content_scan_set_menu_from_playlist_list(playlist_t *playlist, const struct string_list *rdb_list) +{ + return manual_content_scan_set_menu_from_playlist_list_fs(playlist, + rdb_list, NULL); +} + +enum manual_content_scan_playlist_refresh_status + manual_content_scan_set_menu_from_playlist_list_fs( + playlist_t *playlist, const struct string_list *rdb_list, + const manual_content_scan_fs_t *fs) { const char *playlist_path = NULL; const char *content_dir = NULL; @@ -814,7 +855,7 @@ enum manual_content_scan_playlist_refresh_status return MANUAL_CONTENT_SCAN_PLAYLIST_REFRESH_INVALID_SYSTEM_NAME; /* Set content directory */ - if (!manual_content_scan_set_menu_content_dir(content_dir)) + if (!manual_content_scan_set_menu_content_dir_fs(content_dir, fs)) return MANUAL_CONTENT_SCAN_PLAYLIST_REFRESH_INVALID_CONTENT_DIR; if (!manual_content_scan_set_menu_scan_use_db((enum manual_content_scan_db_usage)db_usage)) @@ -922,7 +963,7 @@ enum manual_content_scan_playlist_refresh_status fill_pathname_expand_special(scan_dat_file_path, dat_file_path, sizeof(scan_dat_file_path)); - switch (manual_content_scan_validate_dat_file_path()) + switch (manual_content_scan_validate_dat(fs ? &fs->dat_size : NULL)) { case MANUAL_CONTENT_SCAN_DAT_FILE_INVALID: return MANUAL_CONTENT_SCAN_PLAYLIST_REFRESH_INVALID_DAT_FILE; @@ -1062,11 +1103,57 @@ static unsigned manual_content_scan_match_cores( return count; } +/* Whether @core is written as a path: display names hold + * slashes too, so a path that matches no core falls through + * to the names */ +static bool manual_content_scan_core_is_path(const char *core) +{ + return strchr(core, '/') +#ifdef _WIN32 + || strchr(core, '\\') +#endif + || *core == '~'; +} + +/* @core, a path, as the core list holds paths: expanded, and + * "//" and ".." folded in an absolute path too; "" when it + * resolves to nothing. Asks the file system. */ +static void manual_content_scan_core_path_resolve(char *s, + const char *core, size_t len) +{ + fill_pathname_expand_special(s, core, len); +#ifndef RARCH_CONSOLE + if ( *s + && !path_resolve_realpath(s, len, false) + && !path_is_absolute(s)) + *s = '\0'; +#endif +#ifdef _WIN32 + pathname_conform_slashes_to_os(s); +#endif +} + +/* An override DAT path as the scan opens it: expanded, and a + * relative one resolved; "" when it resolves to nothing. + * Asks the file system. */ +static void manual_content_scan_dat_path_resolve(char *s, + const char *dat, size_t len) +{ + fill_pathname_expand_special(s, dat, len); +#ifndef RARCH_CONSOLE + if ( *s && !path_is_absolute(s) + && !path_resolve_realpath(s, len, false)) + *s = '\0'; +#endif +} + /* Finds the core @core names: a path, else a file name or * file id, else a display name. The first of these that - * matches decides, and *count gets its matches. */ + * matches decides, and *count gets its matches. + * > @resolved: @core resolved as a path already, or NULL to + * resolve it here */ static const core_info_t *manual_content_scan_find_core( - const char *core, unsigned *count) + const char *core, const char *resolved, unsigned *count) { core_info_list_t *core_info_list = NULL; const core_info_t *found = NULL; @@ -1079,25 +1166,12 @@ static const core_info_t *manual_content_scan_find_core( if (!core_info_list || !(s = (char*)malloc(PATH_MAX_LENGTH))) return NULL; - /* Display names hold slashes too, so a path - * that matches no core falls through */ - if ( strchr(core, '/') -#ifdef _WIN32 - || strchr(core, '\\') -#endif - || *core == '~') + if (manual_content_scan_core_is_path(core)) { - fill_pathname_expand_special(s, core, PATH_MAX_LENGTH); -#ifndef RARCH_CONSOLE - /* Folds "//" and ".." in an absolute path too */ - if ( *s - && !path_resolve_realpath(s, PATH_MAX_LENGTH, false) - && !path_is_absolute(s)) - *s = '\0'; -#endif -#ifdef _WIN32 - pathname_conform_slashes_to_os(s); -#endif + if (resolved) + strlcpy(s, resolved, PATH_MAX_LENGTH); + else + manual_content_scan_core_path_resolve(s, core, PATH_MAX_LENGTH); if (*s) *count = manual_content_scan_match_cores(core_info_list, MANUAL_CONTENT_SCAN_CORE_KEY_PATH, s, &found); @@ -1154,6 +1228,74 @@ void manual_content_scan_overrides_free(manual_content_scan_overrides_t *ov) free(ov); } +manual_content_scan_fs_t *manual_content_scan_fs_read( + const char *content_dir, const char *dat, + const char *ov_core, const char *ov_dat) +{ + manual_content_scan_fs_t *fs = (manual_content_scan_fs_t*)calloc(1, + sizeof(*fs)); + char *s = (char*)malloc(PATH_MAX_LENGTH); + + if (!fs || !s) + goto error; + + /* The folder as manual_content_scan_set_menu_content_dir() + * stats it: expanded, one trailing slash gone */ + if (content_dir && *content_dir) + { + size_t _len = fill_pathname_expand_special(s, content_dir, + PATH_MAX_LENGTH); + if (_len > 0 && _len < PATH_MAX_LENGTH + && s[_len - 1] == PATH_DEFAULT_SLASH_C()) + s[_len - 1] = '\0'; + if (*s) + fs->content_stat = path_stat(s); + } + + /* The saved DAT, as a refresh expands it */ + fs->dat_size = -1; + if (dat && *dat) + { + fill_pathname_expand_special(s, dat, PATH_MAX_LENGTH); + if (*s && path_is_valid(s)) + fs->dat_size = path_get_size(s); + } + + fs->ov_dat_size = -1; + if (ov_dat) + { + manual_content_scan_dat_path_resolve(s, ov_dat, PATH_MAX_LENGTH); + if (!(fs->ov_dat = strdup(s))) + goto error; + if (*s && path_is_valid(s)) + fs->ov_dat_size = path_get_size(s); + } + + if (ov_core && manual_content_scan_core_is_path(ov_core)) + { + manual_content_scan_core_path_resolve(s, ov_core, PATH_MAX_LENGTH); + if (!(fs->ov_core = strdup(s))) + goto error; + } + + free(s); + return fs; + +error: + free(s); + manual_content_scan_fs_free(fs); + return NULL; +} + +void manual_content_scan_fs_free(manual_content_scan_fs_t *fs) +{ + if (!fs) + return; + free(fs->ov_dat); + free(fs->ov_core); + free(fs); +} + /* Lays @ov over the current settings and sets the Custom * method, stopping at the first override that does not * resolve; NULL @ov changes nothing. @@ -1163,6 +1305,16 @@ enum manual_content_scan_override_status manual_content_scan_apply_overrides( const manual_content_scan_overrides_t *ov, const struct string_list *rdb_list, unsigned *count) +{ + return manual_content_scan_apply_overrides_fs(ov, rdb_list, count, + NULL); +} + +enum manual_content_scan_override_status + manual_content_scan_apply_overrides_fs( + const manual_content_scan_overrides_t *ov, + const struct string_list *rdb_list, unsigned *count, + const manual_content_scan_fs_t *fs) { if (count) *count = 0; @@ -1174,7 +1326,7 @@ enum manual_content_scan_override_status { unsigned matches = 0; const core_info_t *core_info = manual_content_scan_find_core( - ov->core, &matches); + ov->core, fs ? fs->ov_core : NULL, &matches); if (count) *count = matches; @@ -1232,15 +1384,14 @@ enum manual_content_scan_override_status && ov->match == MANUAL_CONTENT_SCAN_USE_DB_NONE) return MANUAL_CONTENT_SCAN_OVR_DAT_WITHOUT_MATCH; - fill_pathname_expand_special(scan_dat_file_path, ov->dat, - sizeof(scan_dat_file_path)); -#ifndef RARCH_CONSOLE - if ( *scan_dat_file_path && !path_is_absolute(scan_dat_file_path) - && !path_resolve_realpath(scan_dat_file_path, - sizeof(scan_dat_file_path), false)) - scan_dat_file_path[0] = '\0'; -#endif - switch (manual_content_scan_validate_dat_file_path()) + if (fs && fs->ov_dat) + strlcpy(scan_dat_file_path, fs->ov_dat, + sizeof(scan_dat_file_path)); + else + manual_content_scan_dat_path_resolve(scan_dat_file_path, ov->dat, + sizeof(scan_dat_file_path)); + switch (manual_content_scan_validate_dat( + (fs && fs->ov_dat) ? &fs->ov_dat_size : NULL)) { case MANUAL_CONTENT_SCAN_DAT_FILE_OK: break; diff --git a/manual_content_scan.h b/manual_content_scan.h index 002dee24845d..44aa929cd861 100644 --- a/manual_content_scan.h +++ b/manual_content_scan.h @@ -274,6 +274,33 @@ void manual_content_scan_restore_settings(manual_content_scan_saved_t *saved); * Returns true if content directory is valid. */ bool manual_content_scan_set_menu_content_dir(const char *content_dir); +/* What the file system says of one scan's paths, read off the main + * thread by manual_content_scan_fs_read(), so that the _fs setters + * below ask it nothing. Only the paths it was read for: it holds no + * reference to the settings. */ +typedef struct manual_content_scan_fs +{ + char *ov_dat; /* @ov_dat expanded and resolved, or NULL */ + char *ov_core; /* @ov_core resolved when it is a path, or NULL */ + int64_t dat_size; /* the saved DAT's size; <= 0: not there */ + int64_t ov_dat_size; /* likewise, @ov_dat's */ + int content_stat; /* path_stat() of the content folder */ +} manual_content_scan_fs_t; + +/* Any thread: reads nothing but its arguments and the file system. + * @content_dir and @dat as the settings hold them, before expansion; + * @ov_core and @ov_dat an item's overrides, each NULL when unset. + * NULL out of memory. Free with manual_content_scan_fs_free(). */ +manual_content_scan_fs_t *manual_content_scan_fs_read( + const char *content_dir, const char *dat, + const char *ov_core, const char *ov_dat); +void manual_content_scan_fs_free(manual_content_scan_fs_t *fs); + +/* As manual_content_scan_set_menu_content_dir(), the folder's stat + * taken from @fs (NULL: asked now) */ +bool manual_content_scan_set_menu_content_dir_fs(const char *content_dir, + const manual_content_scan_fs_t *fs); + bool manual_content_scan_set_menu_scan_method( enum manual_content_scan_method method); @@ -321,6 +348,13 @@ enum manual_content_scan_playlist_refresh_status manual_content_scan_set_menu_from_playlist_list(playlist_t *playlist, const struct string_list *rdb_list); +/* As manual_content_scan_set_menu_from_playlist_list(), the folder + * and the DAT as @fs read them (NULL: asked now) */ +enum manual_content_scan_playlist_refresh_status + manual_content_scan_set_menu_from_playlist_list_fs( + playlist_t *playlist, const struct string_list *rdb_list, + const manual_content_scan_fs_t *fs); + manual_content_scan_overrides_t *manual_content_scan_overrides_new(void); void manual_content_scan_overrides_free(manual_content_scan_overrides_t *ov); @@ -334,6 +368,14 @@ enum manual_content_scan_override_status const manual_content_scan_overrides_t *ov, const struct string_list *rdb_list, unsigned *count); +/* As manual_content_scan_apply_overrides(), the core path and the + * DAT as @fs read them for @ov (NULL: asked now) */ +enum manual_content_scan_override_status + manual_content_scan_apply_overrides_fs( + const manual_content_scan_overrides_t *ov, + const struct string_list *rdb_list, unsigned *count, + const manual_content_scan_fs_t *fs); + /* Menu getters */ /* Fetches content directory for next manual scan diff --git a/samples/tasks/database/Makefile b/samples/tasks/database/Makefile index 23d3b140fa72..6d0799fbeff0 100644 --- a/samples/tasks/database/Makefile +++ b/samples/tasks/database/Makefile @@ -251,8 +251,9 @@ $(INDEX_TEST)$(EXE_EXT): $(INDEX_TEST_OBJECTS) # query, order the database and content lists, count reader passes, # hold chosen lookups, count an archive's member hits a step, hold # a chosen index build in one step, tell which thread the refresh -# walks and playlist loads ran on, fail a chosen task_init and fail the -# playlist folder walk. +# walks and playlist loads ran on, fail a chosen task_init, fail the +# playlist folder walk, and count the file system calls the main +# thread makes. SCAN_TEST_WRAPS := -Wl,--wrap=libretrodb_open -Wl,--wrap=task_queue_park \ -Wl,--wrap=intfstream_crc_step -Wl,--wrap=rm3u_is_m3u \ -Wl,--wrap=task_nbio_slice_open -Wl,--wrap=task_nbio_slice_close \ @@ -268,7 +269,10 @@ SCAN_TEST_WRAPS := -Wl,--wrap=libretrodb_open -Wl,--wrap=task_queue_park \ -Wl,--wrap=task_nbio_slice_within_budget \ -Wl,--wrap=fill_pathname_join_delim \ -Wl,--wrap=dir_list_iter_new -Wl,--wrap=dir_list_iter_step \ - -Wl,--wrap=playlist_parse_begin_header -Wl,--wrap=task_init + -Wl,--wrap=playlist_parse_begin_header -Wl,--wrap=task_init \ + -Wl,--wrap=path_stat -Wl,--wrap=path_is_valid \ + -Wl,--wrap=path_is_directory -Wl,--wrap=path_get_size \ + -Wl,--wrap=path_resolve_realpath ifeq ($(HAVE_THREADS), 1) SCAN_TEST_WRAPS += -Wl,--wrap=sthread_create endif diff --git a/samples/tasks/database/database_scan_test.c b/samples/tasks/database/database_scan_test.c index 3058792e9204..6af7ea9ea80b 100644 --- a/samples/tasks/database/database_scan_test.c +++ b/samples/tasks/database/database_scan_test.c @@ -10047,6 +10047,149 @@ static void refresh_lane(const char *root, const char *pl_dir, sizeof(config_get_ptr()->paths.directory_playlist)); } +/* File system calls on the main thread while fs_main_watch is set: + * the scan chain's must all run in its prep tasks. Only the main + * thread reads the flag. */ +static bool fs_main_watch; +static unsigned fs_main_calls; +static char fs_main_last[64]; + +static void fs_main_note(const char *fn) +{ +#ifdef HAVE_THREADS + if (current_tid() == main_tid && fs_main_watch) + { + fs_main_calls++; + strlcpy(fs_main_last, fn, sizeof(fs_main_last)); + } +#else + (void)fn; +#endif +} + +int __real_path_stat(const char *path); +int __wrap_path_stat(const char *path) +{ + fs_main_note("path_stat"); + return __real_path_stat(path); +} + +bool __real_path_is_valid(const char *path); +bool __wrap_path_is_valid(const char *path) +{ + fs_main_note("path_is_valid"); + return __real_path_is_valid(path); +} + +bool __real_path_is_directory(const char *path); +bool __wrap_path_is_directory(const char *path) +{ + fs_main_note("path_is_directory"); + return __real_path_is_directory(path); +} + +int64_t __real_path_get_size(const char *path); +int64_t __wrap_path_get_size(const char *path) +{ + fs_main_note("path_get_size"); + return __real_path_get_size(path); +} + +char *__real_path_resolve_realpath(char *s, size_t len, + bool resolve_symlinks); +char *__wrap_path_resolve_realpath(char *s, size_t len, + bool resolve_symlinks) +{ + fs_main_note("path_resolve_realpath"); + return __real_path_resolve_realpath(s, len, resolve_symlinks); +} + +/* On a threaded queue the chain asks the file system nothing on the + * main thread before a scan: a --scan path's resolve and stat, an + * override core path's and DAT's, a --scan-refresh path's realpaths + * and its content folder's stat all run on the worker. Every item + * here stops before its scan, so the whole run is watched. */ +static void fs_main_lane(const char *root, const char *pl_dir) +{ +#ifdef HAVE_THREADS + char pl[512], a[512], gone[512], outside[512]; + char via[1024], p[1024], stale[1024], detail[256]; + task_dbscan_item_t *items; + bool pushed; + + sprintf(pl, "%s/fsmain_pl", root); + sprintf(a, "%s/fsmain_a", root); + sprintf(gone, "%s/fsmain_gone", root); + sprintf(outside, "%s/fsmain_out", root); + sprintf(stale, "%s/00_gone.bin", a); + path_mkdir(pl); + path_mkdir(a); + path_mkdir(outside); + delete_playlists(pl); + sprintf(p, "%s/f_gone.lpl", pl); + if (!refresh_write(p, gone, NULL, stale)) + goto fixture; + sprintf(p, "%s/f_none.lpl", pl); + if (!refresh_write(p, NULL, NULL, stale)) + goto fixture; + sprintf(p, "%s/x.lpl", outside); + if (!refresh_write(p, a, NULL, stale)) + goto fixture; + /* In the folder, but by a path that only a realpath folds */ + sprintf(via, "%s/../%s/f_gone.lpl", pl, path_basename(pl)); + strlcpy(config_get_ptr()->paths.directory_playlist, pl, + sizeof(config_get_ptr()->paths.directory_playlist)); + use_queue(true); + + if (!(items = (task_dbscan_item_t*)calloc(6, sizeof(*items)))) + goto fixture; + chain_item(&items[0], "fsmain_missing_dir", NULL); + chain_item(&items[1], a, "./fsmain_nocore_libretro.so"); + chain_item(&items[2], a, NULL); + if ((items[2].overrides = manual_content_scan_overrides_new())) + items[2].overrides->dat = ovr_str("fsmain_missing.dat"); + refresh_item(&items[3], TASK_DBSCAN_ITEM_REFRESH, via); + refresh_item(&items[4], TASK_DBSCAN_ITEM_REFRESH, p); + refresh_item(&items[5], TASK_DBSCAN_ITEM_REFRESH, "f_none"); + + fs_main_calls = 0; + fs_main_last[0] = '\0'; + fs_main_watch = true; + pushed = chain_push(items, 6); + chain_wait(false); + fs_main_watch = false; + + snprintf(detail, sizeof(detail), "%u items, failed %u, skipped %u", + chain_seen, chain_totals.failed, chain_totals.skipped); + check( pushed && chain_dones == 1 && chain_seen == 6 + && chain_saw(0, "fsmain_missing_dir", TASK_DBSCAN_FAILED, -1) + && chain_saw(1, a, TASK_DBSCAN_FAILED, -1) + && chain_saw(2, a, TASK_DBSCAN_FAILED, -1) + && chain_saw(3, "f_gone", TASK_DBSCAN_SKIPPED, -1) + && chain_saw(4, p, TASK_DBSCAN_FAILED, -1) + && chain_saw(5, "f_none", TASK_DBSCAN_FAILED, -1) + && chain_totals.failed == 5 && chain_totals.skipped == 1, + "fs off main: each item stops where it did", detail); + snprintf(detail, sizeof(detail), "%u calls, the last %s", + fs_main_calls, *fs_main_last ? fs_main_last : "none"); + check(fs_main_calls == 0, + "fs off main: no file system call on the main thread", detail); + + manual_content_scan_reset_settings(); + strlcpy(config_get_ptr()->paths.directory_playlist, pl_dir, + sizeof(config_get_ptr()->paths.directory_playlist)); + return; + +fixture: + check(0, "fixture", "could not write the fs off main lane"); + strlcpy(config_get_ptr()->paths.directory_playlist, pl_dir, + sizeof(config_get_ptr()->paths.directory_playlist)); +#else + (void)root; + (void)pl_dir; +#endif +} + /* The summary the menu's Refresh All Playlists shows. */ static char menu_all_msg[128]; @@ -11241,6 +11384,7 @@ int main(int argc, char **argv) chain_lane(root, pl_dir, crc_a, sz_a, true); refresh_lane(root, pl_dir, crc_a, sz_a, false); refresh_lane(root, pl_dir, crc_a, sz_a, true); + fs_main_lane(root, pl_dir); menu_refresh_all_lane(root, pl_dir, crc_a, sz_a, false); menu_refresh_all_lane(root, pl_dir, crc_a, sz_a, true); { diff --git a/tasks/task_database.c b/tasks/task_database.c index 4312c2300677..075d7d08797a 100644 --- a/tasks/task_database.c +++ b/tasks/task_database.c @@ -5882,7 +5882,8 @@ enum dbscan_prep_kind { DBSCAN_PREP_DBS = 0, /* the database folder */ DBSCAN_PREP_LIST, /* the playlist folder */ - DBSCAN_PREP_LOAD /* one playlist */ + DBSCAN_PREP_LOAD, /* one playlist */ + DBSCAN_PREP_PATH /* a --scan path */ }; enum dbscan_load_status @@ -5891,7 +5892,9 @@ enum dbscan_load_status DBSCAN_LOAD_NO_PLAYLIST, DBSCAN_LOAD_AMBIGUOUS, DBSCAN_LOAD_NO_SETTINGS, - DBSCAN_LOAD_FOLDER_MISSING + DBSCAN_LOAD_FOLDER_MISSING, + DBSCAN_LOAD_OUTSIDE, /* a --scan-refresh path not in the folder */ + DBSCAN_LOAD_NO_MEMORY }; /* A prep task's state: the handler fills it, and the callback takes @@ -5901,6 +5904,16 @@ typedef struct playlist_config_t config; /* LOAD: the playlist */ char *dir; /* DBS, LIST; a --scan-refresh's LOAD */ char *name; /* LOAD: the name its lines give */ + /* PATH, and a --scan-refresh's LOAD: the item as typed; that LOAD's + * playlist folder as configured. Copies, as are the overrides' + * core and DAT: the worker reads nothing of the chain. */ + char *arg; + char *pl_dir; + char *ov_core; + char *ov_dat; + char *path; /* PATH: resolved, NULL when not there */ + char *detail; /* LOAD: OUTSIDE's folder, for its line */ + manual_content_scan_fs_t *fs; /* PATH, LOAD: what the file system said */ dir_list_iter_t *iter; struct string_list *list; playlist_parse_t *parse; @@ -5910,7 +5923,7 @@ typedef struct unsigned matches; /* LOAD: files in another case, if ambiguous */ bool include_hidden; bool begun; - bool failed; /* LIST: out of memory */ + bool failed; /* LIST, PATH: out of memory */ } dbscan_prep_t; typedef struct @@ -6010,24 +6023,34 @@ static char *dbscan_chain_expand(const char *in) return s; } -/* @in expanded; NULL, with the reason on stderr, if nothing is - * there. Free the result. */ -static char *dbscan_chain_resolve(const char *in) +/* Resolves a --scan path and reads what its scan will ask of the + * file system: on a threaded queue's worker, else at once on the main + * thread, as a stat is one call. */ +static void dbscan_path_handler(retro_task_t *task) { - char *s = dbscan_chain_expand(in); + dbscan_prep_t *prep = (dbscan_prep_t*)task->state; - if (!s) + if (!(task_get_flags(task) & RETRO_TASK_FLG_CANCELLED)) { - fprintf(stderr, "out of memory\n"); - return NULL; - } - if (!*s || !path_is_valid(s)) - { - fprintf(stderr, "%s: not found\n", in); - free(s); - return NULL; + char *s = dbscan_chain_expand(prep->arg); + + if (!s) + prep->failed = true; + else if (!*s) + free(s); + else if (!(prep->fs = manual_content_scan_fs_read(s, NULL, + prep->ov_core, prep->ov_dat))) + { + prep->failed = true; + free(s); + } + /* The stat the content folder's setter takes */ + else if (!(prep->fs->content_stat & RETRO_VFS_STAT_IS_VALID)) + free(s); + else + prep->path = s; } - return s; + task_set_flags(task, RETRO_TASK_FLG_FINISHED, true); } /* @name: the item, for the one error the overrides cannot name. */ @@ -6126,24 +6149,29 @@ static void dbscan_chain_single_warn(const dbscan_chain_t *chain, name); } -/* True once the item's scan is pushed; otherwise it is folded as - * failed. */ -static bool dbscan_chain_push_path(dbscan_chain_t *chain, - const task_dbscan_item_t *item) +/* True once the item's scan is pushed, @prep's path and what the file + * system said of it in hand; otherwise it is folded as failed. */ +static bool dbscan_chain_push_scan(dbscan_chain_t *chain, + const task_dbscan_item_t *item, dbscan_prep_t *prep) { enum manual_content_scan_override_status st; unsigned count = 0; - char *path = dbscan_chain_resolve(item->arg); + char *path = prep->path; if (!path) { + if (prep->failed) + fprintf(stderr, "out of memory\n"); + else + fprintf(stderr, "%s: not found\n", item->arg); dbscan_chain_fold(chain, item->arg, TASK_DBSCAN_FAILED, NULL); return false; } + prep->path = NULL; manual_content_scan_reset_settings(); - manual_content_scan_set_menu_content_dir(path); - if ((st = manual_content_scan_apply_overrides(item->overrides, - chain->rdb_list, &count)) != MANUAL_CONTENT_SCAN_OVR_OK) + manual_content_scan_set_menu_content_dir_fs(path, prep->fs); + if ((st = manual_content_scan_apply_overrides_fs(item->overrides, + chain->rdb_list, &count, prep->fs)) != MANUAL_CONTENT_SCAN_OVR_OK) dbscan_chain_override_error(chain, item->arg, item->overrides, st, count); else @@ -6152,7 +6180,7 @@ static bool dbscan_chain_push_path(dbscan_chain_t *chain, if ( chain->cli_output && (flags & MANUAL_CONTENT_SCAN_OVR_OVERWRITE) - && !path_is_directory(path)) + && !(prep->fs->content_stat & RETRO_VFS_STAT_IS_DIRECTORY)) { fprintf(stderr, "%s: --scan-overwrite ignored for a single file\n", path); @@ -6184,8 +6212,8 @@ static bool dbscan_chain_push_refreshed(dbscan_chain_t *chain, playlist_t *pl = prep->playlist; manual_content_scan_reset_settings(); - switch (manual_content_scan_set_menu_from_playlist_list(pl, - chain->rdb_list)) + switch (manual_content_scan_set_menu_from_playlist_list_fs(pl, + chain->rdb_list, prep->fs)) { case MANUAL_CONTENT_SCAN_PLAYLIST_REFRESH_OK: break; @@ -6219,8 +6247,9 @@ static bool dbscan_chain_push_refreshed(dbscan_chain_t *chain, prep->playlist = NULL; if (!reason) { - if ((st = manual_content_scan_apply_overrides(item->overrides, - chain->rdb_list, &count)) != MANUAL_CONTENT_SCAN_OVR_OK) + if ((st = manual_content_scan_apply_overrides_fs(item->overrides, + chain->rdb_list, &count, prep->fs)) + != MANUAL_CONTENT_SCAN_OVR_OK) dbscan_chain_override_error(chain, prep->name, item->overrides, st, count); else @@ -6402,6 +6431,91 @@ static bool dbscan_load_name_step(dbscan_prep_t *prep) return true; } +/* @prep loads @path: its name, the path, and, when @list_dir, the + * folder to list for the name's case. Strings only, any thread. */ +static void dbscan_prep_set_playlist(dbscan_prep_t *prep, + const char *path, bool list_dir) +{ + fill_pathname(prep->name, path_basename(path), "", NAME_MAX_LENGTH); + if (list_dir && (prep->dir = (char*)malloc(PATH_MAX_LENGTH))) + { + fill_pathname_basedir(prep->dir, path, PATH_MAX_LENGTH); +#ifdef _WIN32 + pathname_conform_slashes_to_os(prep->dir); +#endif + } + playlist_config_set_path(&prep->config, path); +} + +/* The playlist a --scan-refresh's @prep->arg names in @prep->pl_dir: + * false, with the status set, when it names none there. On the + * load's worker, as the realpaths ask the file system. */ +static bool dbscan_prep_refresh_resolve(dbscan_prep_t *prep) +{ + const char *arg = prep->arg; + size_t _len = strlen(arg); + char *path = NULL; + char *dir = NULL; + char *want = NULL; + bool ok = false; + + /* A playlist name cannot hold a slash; a file name is one too. */ + if (!find_last_slash(arg)) + { + if ((path = (char*)malloc(PATH_MAX_LENGTH))) + { + fill_pathname_join_special(path, prep->pl_dir, arg, + PATH_MAX_LENGTH); + if (!( _len >= STRLEN_CONST(".lpl") + && string_is_equal_case_insensitive( + arg + _len - STRLEN_CONST(".lpl"), ".lpl"))) + strlcat(path, ".lpl", PATH_MAX_LENGTH); + dbscan_prep_set_playlist(prep, path, true); + ok = true; + } + else + prep->load = DBSCAN_LOAD_NO_MEMORY; + free(path); + return ok; + } + path = dbscan_chain_expand(arg); + dir = dbscan_chain_expand(prep->pl_dir); + want = (char*)malloc(PATH_MAX_LENGTH); + if (path && dir && want) + { + /* "//", "/./" and ".." would fail the comparison; @dir stays + * as configured for the line. */ + path_resolve_realpath(path, PATH_MAX_LENGTH, false); + /* The scan writes into the playlist folder, whatever the path. */ + fill_pathname_join_special(want, dir, path_basename(path), + PATH_MAX_LENGTH); + path_resolve_realpath(want, PATH_MAX_LENGTH, false); +#ifdef _WIN32 + pathname_conform_slashes_to_os(path); + pathname_conform_slashes_to_os(want); + if (string_is_equal_case_insensitive(path, want)) +#else + if (string_is_equal(path, want)) +#endif + { + dbscan_prep_set_playlist(prep, path, true); + ok = true; + } + else + { + prep->load = DBSCAN_LOAD_OUTSIDE; + prep->detail = dir; + dir = NULL; + } + } + else + prep->load = DBSCAN_LOAD_NO_MEMORY; + free(path); + free(dir); + free(want); + return ok; +} + /* Loads one playlist and tells whether it can be refreshed: to the end * on a threaded queue's worker, else under the shared window. */ static void dbscan_load_handler(retro_task_t *task) @@ -6416,6 +6530,12 @@ static void dbscan_load_handler(retro_task_t *task) if (!prep->begun) { prep->begun = true; + /* A --scan-refresh: its path first. */ + if (prep->arg && !dbscan_prep_refresh_resolve(prep)) + { + task_set_flags(task, RETRO_TASK_FLG_FINISHED, true); + return; + } /* The parse waits for the listed path. */ if (prep->dir && (prep->list = string_list_new())) prep->iter = dir_list_iter_new(prep->dir, "lpl", false, true, @@ -6470,21 +6590,19 @@ static void dbscan_load_handler(retro_task_t *task) } if (prep->load == DBSCAN_LOAD_OK) { - char *dir = NULL; - /* A playlist that does not parse has no settings to read. */ if (!playlist_scan_refresh_enabled(prep->playlist)) prep->load = DBSCAN_LOAD_NO_SETTINGS; - else if ((dir = (char*)malloc(PATH_MAX_LENGTH))) - { - fill_pathname_expand_special(dir, - playlist_get_scan_content_dir(prep->playlist), - PATH_MAX_LENGTH); - /* A single file's scan saves the file. */ - if (!path_is_valid(dir)) - prep->load = DBSCAN_LOAD_FOLDER_MISSING; - free(dir); - } + /* Here, so that the refresh asks the file system nothing on the + * main thread: the folder, the saved DAT, the overrides'. */ + else if (!(prep->fs = manual_content_scan_fs_read( + playlist_get_scan_content_dir(prep->playlist), + playlist_get_scan_dat_file_path(prep->playlist), + prep->ov_core, prep->ov_dat))) + prep->load = DBSCAN_LOAD_NO_MEMORY; + /* A single file's scan saves the file. */ + else if (!(prep->fs->content_stat & RETRO_VFS_STAT_IS_VALID)) + prep->load = DBSCAN_LOAD_FOLDER_MISSING; } task_set_flags(task, RETRO_TASK_FLG_FINISHED, true); } @@ -6532,6 +6650,16 @@ static void dbscan_load_cb(retro_task_t *task, void *task_data, NULL); dbscan_chain_fold(chain, arg, TASK_DBSCAN_FAILED, NULL); break; + case DBSCAN_LOAD_OUTSIDE: + dbscan_chain_refresh_error(chain, arg, + "not in the playlist folder", prep->detail); + dbscan_chain_fold(chain, arg, TASK_DBSCAN_FAILED, NULL); + break; + case DBSCAN_LOAD_NO_MEMORY: + if (chain->cli_output) + fprintf(stderr, "out of memory\n"); + dbscan_chain_fold(chain, arg, TASK_DBSCAN_FAILED, NULL); + break; case DBSCAN_LOAD_FOLDER_MISSING: dbscan_chain_refresh_error(chain, prep->name, "skipped: content folder not found", @@ -6546,10 +6674,8 @@ static void dbscan_load_cb(retro_task_t *task, void *task_data, dbscan_chain_next(chain); } -static void dbscan_prep_free(retro_task_t *task) +static void dbscan_prep_state_free(dbscan_prep_t *prep) { - dbscan_prep_t *prep = (dbscan_prep_t*)task->state; - if (!prep) return; dir_list_iter_free(prep->iter); @@ -6557,33 +6683,64 @@ static void dbscan_prep_free(retro_task_t *task) string_list_free(prep->list); playlist_parse_abort(prep->parse); playlist_free(prep->playlist); + manual_content_scan_fs_free(prep->fs); free(prep->name); free(prep->dir); + free(prep->arg); + free(prep->pl_dir); + free(prep->ov_core); + free(prep->ov_dat); + free(prep->path); + free(prep->detail); free(prep); } +static void dbscan_prep_free(retro_task_t *task) +{ + dbscan_prep_state_free((dbscan_prep_t*)task->state); +} + +/* Copies @item's core and DAT overrides into @prep, for its worker; + * false out of memory. */ +static bool dbscan_prep_take_overrides(dbscan_prep_t *prep, + const task_dbscan_item_t *item) +{ + const manual_content_scan_overrides_t *ov = item->overrides; + + if (!ov) + return true; + if (ov->core && !(prep->ov_core = strdup(ov->core))) + return false; + if (ov->dat && !(prep->ov_dat = strdup(ov->dat))) + return false; + return true; +} + /* Pushes a prep task for @prep; false, @prep freed, if it cannot. */ static bool dbscan_chain_push_prep(dbscan_chain_t *chain, - dbscan_prep_t *prep, bool load, retro_task_callback_t cb) + dbscan_prep_t *prep, retro_task_callback_t cb) { retro_task_t *task = prep ? task_init() : NULL; if (!task) { - if (prep) - { - free(prep->name); - free(prep->dir); - free(prep); - } + dbscan_prep_state_free(prep); return false; } /* Silent: no title, no notifications. Each handler by name, so the - * thread-read audit takes both as worker entries. */ - if (load) - task->handler = dbscan_load_handler; - else - task->handler = dbscan_prep_handler; + * thread-read audit takes them all as worker entries. */ + switch (prep->kind) + { + case DBSCAN_PREP_LOAD: + task->handler = dbscan_load_handler; + break; + case DBSCAN_PREP_PATH: + task->handler = dbscan_path_handler; + break; + default: + task->handler = dbscan_prep_handler; + break; + } task->state = prep; task->user_data = chain; task->callback = cb; @@ -6592,34 +6749,26 @@ static bool dbscan_chain_push_prep(dbscan_chain_t *chain, return true; } -/* Loads the playlist at @path off the main thread; otherwise folds it - * as failed. */ -static bool dbscan_chain_push_load(dbscan_chain_t *chain, - const char *path) +/* A load prep for the current item, its playlist not yet set: NULL + * out of memory. */ +static dbscan_prep_t *dbscan_prep_load_new(dbscan_chain_t *chain) { - char name[NAME_MAX_LENGTH]; settings_t *settings = config_get_ptr(); dbscan_prep_t *prep = (dbscan_prep_t*)calloc(1, sizeof(*prep)); - if (prep && !(prep->name = (char*)malloc(NAME_MAX_LENGTH))) + if (!prep) + return NULL; + prep->kind = DBSCAN_PREP_LOAD; + if ( !(prep->name = (char*)malloc(NAME_MAX_LENGTH)) + || !dbscan_prep_take_overrides(prep, + &chain->items[chain->next - 1])) { - free(prep); - prep = NULL; + dbscan_prep_state_free(prep); + return NULL; } - if (prep) + *prep->name = '\0'; { - fill_pathname(prep->name, path_basename(path), "", NAME_MAX_LENGTH); - /* A refresh-all's names come from its listing already. */ - if ( chain->items[chain->next - 1].kind == TASK_DBSCAN_ITEM_REFRESH - && (prep->dir = (char*)malloc(PATH_MAX_LENGTH))) - { - fill_pathname_basedir(prep->dir, path, PATH_MAX_LENGTH); -#ifdef _WIN32 - pathname_conform_slashes_to_os(prep->dir); -#endif - } /* As Manage Playlists opens it. */ - playlist_config_set_path(&prep->config, path); prep->config.capacity = COLLECTION_SIZE; prep->config.old_format = settings->bools.playlist_use_old_format; prep->config.compress = settings->bools.playlist_compression; @@ -6627,10 +6776,22 @@ static bool dbscan_chain_push_load(dbscan_chain_t *chain, playlist_config_set_base_content_directory(&prep->config, settings->bools.playlist_portable_paths ? settings->paths.directory_menu_content : NULL); - prep->kind = DBSCAN_PREP_LOAD; } - if (dbscan_chain_push_prep(chain, prep, true, - dbscan_load_cb)) + return prep; +} + +/* Loads a refresh-all's playlist at @path off the main thread; + * otherwise folds it as failed. Its names come from its listing + * already. */ +static bool dbscan_chain_push_load(dbscan_chain_t *chain, + const char *path) +{ + char name[NAME_MAX_LENGTH]; + dbscan_prep_t *prep = dbscan_prep_load_new(chain); + + if (prep) + dbscan_prep_set_playlist(prep, path, false); + if (dbscan_chain_push_prep(chain, prep, dbscan_load_cb)) return true; fill_pathname(name, path_basename(path), "", sizeof(name)); dbscan_chain_refresh_error(chain, name, "the scan could not start", NULL); @@ -6638,84 +6799,69 @@ static bool dbscan_chain_push_load(dbscan_chain_t *chain, return false; } -/* The playlist @arg names, or NULL, with its line, if it names none - * in the playlist folder. Free the result. */ -static char *dbscan_chain_refresh_path(const dbscan_chain_t *chain, - const char *arg) +/* True once @item's playlist is loading; otherwise it is folded as + * failed. */ +static bool dbscan_chain_push_refresh(dbscan_chain_t *chain, + const task_dbscan_item_t *item) { - const char *pl_dir = config_get_ptr()->paths.directory_playlist; - size_t _len = strlen(arg); - char *path = NULL; - char *dir = NULL; - char *want = NULL; + dbscan_prep_t *prep = dbscan_prep_load_new(chain); - /* A playlist name cannot hold a slash; a file name is one too. */ - if (!find_last_slash(arg)) + /* The worker finds the playlist: the realpaths ask the file + * system. */ + if ( prep + && ( !(prep->arg = strdup(item->arg)) + || !(prep->pl_dir = strdup( + config_get_ptr()->paths.directory_playlist)))) { - if ((path = (char*)malloc(PATH_MAX_LENGTH))) - { - fill_pathname_join_special(path, pl_dir, arg, PATH_MAX_LENGTH); - if (!( _len >= STRLEN_CONST(".lpl") - && string_is_equal_case_insensitive( - arg + _len - STRLEN_CONST(".lpl"), ".lpl"))) - strlcat(path, ".lpl", PATH_MAX_LENGTH); - } - else if (chain->cli_output) - fprintf(stderr, "out of memory\n"); - return path; - } - path = dbscan_chain_expand(arg); - dir = dbscan_chain_expand(pl_dir); - want = (char*)malloc(PATH_MAX_LENGTH); - if (path && dir && want) - { - /* "//", "/./" and ".." would fail the comparison; @dir stays - * as configured for the line. */ - path_resolve_realpath(path, PATH_MAX_LENGTH, false); - /* The scan writes into the playlist folder, whatever the path. */ - fill_pathname_join_special(want, dir, path_basename(path), - PATH_MAX_LENGTH); - path_resolve_realpath(want, PATH_MAX_LENGTH, false); -#ifdef _WIN32 - pathname_conform_slashes_to_os(path); - pathname_conform_slashes_to_os(want); - if (!string_is_equal_case_insensitive(path, want)) -#else - if (!string_is_equal(path, want)) -#endif - { - dbscan_chain_refresh_error(chain, arg, - "not in the playlist folder", dir); - free(path); - path = NULL; - } + dbscan_prep_state_free(prep); + prep = NULL; } - else + if (dbscan_chain_push_prep(chain, prep, dbscan_load_cb)) + return true; + dbscan_chain_refresh_error(chain, item->arg, "the scan could not start", + NULL); + dbscan_chain_fold(chain, item->arg, TASK_DBSCAN_FAILED, NULL); + return false; +} + +static void dbscan_path_cb(retro_task_t *task, void *task_data, + void *user_data, const char *err) +{ + dbscan_chain_t *chain = (dbscan_chain_t*)user_data; + dbscan_prep_t *prep = (dbscan_prep_t*)task->state; + + if (task_get_flags(task) & RETRO_TASK_FLG_CANCELLED) { - if (chain->cli_output) - fprintf(stderr, "out of memory\n"); - free(path); - path = NULL; + chain->totals.cancelled = true; + dbscan_chain_finish(chain); + return; } - free(dir); - free(want); - return path; + if (!dbscan_chain_push_scan(chain, &chain->items[chain->next - 1], prep)) + dbscan_chain_next(chain); } -/* True once @item's playlist is loading; otherwise it is folded as +/* True once @item's path is being read; otherwise it is folded as * failed. */ -static bool dbscan_chain_push_refresh(dbscan_chain_t *chain, +static bool dbscan_chain_push_path(dbscan_chain_t *chain, const task_dbscan_item_t *item) { - bool pushed = false; - char *path = dbscan_chain_refresh_path(chain, item->arg); + dbscan_prep_t *prep = (dbscan_prep_t*)calloc(1, sizeof(*prep)); - if (path) - pushed = dbscan_chain_push_load(chain, path); - else - dbscan_chain_fold(chain, item->arg, TASK_DBSCAN_FAILED, NULL); - free(path); - return pushed; + if (prep) + { + prep->kind = DBSCAN_PREP_PATH; + if ( !(prep->arg = strdup(item->arg)) + || !dbscan_prep_take_overrides(prep, item)) + { + dbscan_prep_state_free(prep); + prep = NULL; + } + } + if (dbscan_chain_push_prep(chain, prep, dbscan_path_cb)) + return true; + fprintf(stderr, "out of memory\n"); + dbscan_chain_fold(chain, item->arg, TASK_DBSCAN_FAILED, NULL); + return false; } /* Lists the playlist folder off the main thread when a refresh-all is @@ -6736,8 +6882,7 @@ static bool dbscan_chain_push_list(dbscan_chain_t *chain) prep = NULL; } } - return dbscan_chain_push_prep(chain, prep, false, - dbscan_prep_cb); + return dbscan_chain_push_prep(chain, prep, dbscan_prep_cb); } /* Runs items, and a refresh-all's playlists, until one pushes a task; @@ -6863,16 +7008,13 @@ bool task_push_dbscan_chain(task_dbscan_item_t *items, size_t count, goto error; dbscan_chain_running = chain; - if (dbscan_chain_push_prep(chain, prep, false, - dbscan_prep_cb)) + if (dbscan_chain_push_prep(chain, prep, dbscan_prep_cb)) return true; dbscan_chain_running = NULL; prep = NULL; error: - if (prep) - free(prep->dir); - free(prep); + dbscan_prep_state_free(prep); if (chain) free(chain->saved); free(chain); From 1e9a95c1e9778a6078dedc2af3473e9a5074809d Mon Sep 17 00:00:00 2001 From: LibretroAdmin Date: Wed, 4 Nov 2026 19:15:18 +0100 Subject: [PATCH 21/21] samples: the scan chain's lanes hold on Windows and without realpath From a native Windows run of both tests: - database_scan_test "fs off main" refreshed f_gone.lpl through "pl/../pl/", which only a realpath folds; the sample builds without RARCH_INTERNAL, where path_resolve_realpath() folds nothing, so the item was rightly not in the playlist folder. It names the playlist by its plain path now, which the check still realpaths on the worker. - "another case, a database's playlist" read the system name type after the chain ended, and the chain puts the Manual Scan settings back then. The item callback records each scan's type instead. - playlist_parity_test's header lane expected "/moved/snes"; autofix converts the suffix to the local form, "/moved\snes" on Windows. - The worker lane's stress read required a core name of every read. Windows replaces the file with MoveFileExW, and an open that meets the replace fails: that read is an empty playlist, which has none. --- samples/playlist/playlist_parity_test.c | 19 +++++++++++++++---- samples/tasks/database/database_scan_test.c | 19 +++++++++++++------ 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/samples/playlist/playlist_parity_test.c b/samples/playlist/playlist_parity_test.c index 23751547f401..08c180a27e03 100644 --- a/samples/playlist/playlist_parity_test.c +++ b/samples/playlist/playlist_parity_test.c @@ -890,9 +890,14 @@ static void lane_cache_worker_write(void) { for (i = 0; i < 400; i++) { + /* Windows replaces the file with MoveFileExW, and an open + * that meets it fails: that read is an empty playlist. */ if (playlist_init_cached(&config) && (pl = playlist_get_cached())) - CHECK(playlist_get_default_core_name(pl) != NULL, - "worker lane: stress read"); + CHECK( playlist_get_default_core_name(pl) != NULL +#ifdef _WIN32 + || !playlist_size(pl) +#endif + , "worker lane: stress read"); if (i & 1) playlist_free_cached(); } @@ -978,8 +983,14 @@ static void lane_header_only(void) before = slurp(path, &blen); playlist_config_set_base_content_directory(&config, "/moved"); pl = header_read(&config); - CHECK(pl && streq(playlist_get_scan_content_dir(pl), "/moved/snes"), - "header lane: scan content dir under the moved base"); + /* the suffix in the local form, as autofix converts it */ + CHECK(pl && streq(playlist_get_scan_content_dir(pl), +#ifdef _WIN32 + "/moved\\snes" +#else + "/moved/snes" +#endif + ), "header lane: scan content dir under the moved base"); if (pl) { /* and never written, even when asked */ diff --git a/samples/tasks/database/database_scan_test.c b/samples/tasks/database/database_scan_test.c index 6af7ea9ea80b..4d31da853798 100644 --- a/samples/tasks/database/database_scan_test.c +++ b/samples/tasks/database/database_scan_test.c @@ -9142,6 +9142,9 @@ static bool chain_ud_ok; static char chain_names[CHAIN_SEEN_MAX][512]; static enum task_dbscan_outcome chain_outcomes[CHAIN_SEEN_MAX]; static int chain_added[CHAIN_SEEN_MAX]; +/* The system name type each item's scan ran with: the chain puts the + * Manual Scan settings back once it ends. */ +static unsigned chain_name_types[CHAIN_SEEN_MAX]; static task_dbscan_chain_result_t chain_totals; static void chain_item_cb(const char *name, @@ -9156,6 +9159,8 @@ static void chain_item_cb(const char *name, sizeof(chain_names[0])); chain_outcomes[chain_seen] = outcome; chain_added[chain_seen] = res ? (int)res->added : -1; + chain_name_types[chain_seen] = + manual_content_scan_get_menu_system_name_type(); } chain_seen++; } @@ -9904,8 +9909,10 @@ static void refresh_lane(const char *root, const char *pl_dir, && !file_contains(out, "test beta") && !file_contains(out, "TEST BETA"), what, detail); sprintf(what, "refresh %s: another case, a database's playlist", q); - check( manual_content_scan_get_menu_system_name_type() - == MANUAL_CONTENT_SCAN_SYSTEM_NAME_DATABASE, what, ""); + check( chain_seen == 2 + && chain_name_types[0] == MANUAL_CONTENT_SCAN_SYSTEM_NAME_DATABASE + && chain_name_types[1] == MANUAL_CONTENT_SCAN_SYSTEM_NAME_DATABASE, + what, ""); sprintf(what, "refresh %s: another case keeps the file's name", q); check( listed_as(case_pl, "Test Beta.lpl", detail, sizeof(detail)) && !strcmp(detail, "Test Beta.lpl"), what, detail); @@ -10113,7 +10120,7 @@ static void fs_main_lane(const char *root, const char *pl_dir) { #ifdef HAVE_THREADS char pl[512], a[512], gone[512], outside[512]; - char via[1024], p[1024], stale[1024], detail[256]; + char by_path[1024], p[1024], stale[1024], detail[256]; task_dbscan_item_t *items; bool pushed; @@ -10135,8 +10142,8 @@ static void fs_main_lane(const char *root, const char *pl_dir) sprintf(p, "%s/x.lpl", outside); if (!refresh_write(p, a, NULL, stale)) goto fixture; - /* In the folder, but by a path that only a realpath folds */ - sprintf(via, "%s/../%s/f_gone.lpl", pl, path_basename(pl)); + /* By its path: the check that it is in the folder realpaths both */ + sprintf(by_path, "%s/f_gone.lpl", pl); strlcpy(config_get_ptr()->paths.directory_playlist, pl, sizeof(config_get_ptr()->paths.directory_playlist)); use_queue(true); @@ -10148,7 +10155,7 @@ static void fs_main_lane(const char *root, const char *pl_dir) chain_item(&items[2], a, NULL); if ((items[2].overrides = manual_content_scan_overrides_new())) items[2].overrides->dat = ovr_str("fsmain_missing.dat"); - refresh_item(&items[3], TASK_DBSCAN_ITEM_REFRESH, via); + refresh_item(&items[3], TASK_DBSCAN_ITEM_REFRESH, by_path); refresh_item(&items[4], TASK_DBSCAN_ITEM_REFRESH, p); refresh_item(&items[5], TASK_DBSCAN_ITEM_REFRESH, "f_none");