diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d13fb7b..56cefa0b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,8 @@ I've focused on modded lobby joining and made the Online menus easier to use. - Live offline FOV adjustment, including paused races, with the corrected guest register handling. Existing online restrictions stay. - Windows updater log handoff and better session diagnostics. +- Fixed installed Store mods returning to Needs Attention after restarting. + Existing completed entries are recovered without downloading them again. - Integrated the patch guardrail setup and follow-ups from PRs #76–#78. The full source consolidation remains separate. - Updated current guides and prepared Windows x64, both Linux AppImages and the diff --git a/docs/DKR-R-v1.0.12-Changelog.txt b/docs/DKR-R-v1.0.12-Changelog.txt index 5ca40fb3..4c2cbc18 100644 --- a/docs/DKR-R-v1.0.12-Changelog.txt +++ b/docs/DKR-R-v1.0.12-Changelog.txt @@ -15,6 +15,8 @@ FOV slider so you can see it changing while you play. - Added remembered Discord sign-in on Linux and Steam Deck without a keyring. - Made the offline FOV slider work live, including while paused. - Fixed logging after an updater restart and improved join diagnostics. +- Fixed installed Store mods showing up in Needs Attention after a restart. + You won't need to download those installed mods again. All the existing memory, HUD, modded lobby and spectator fixes are kept. Your saves, mods and settings can stay. Please use the complete v1.0.12 diff --git a/docs/PUBLIC-1.0.12-NOTES.txt b/docs/PUBLIC-1.0.12-NOTES.txt index 5ca40fb3..4c2cbc18 100644 --- a/docs/PUBLIC-1.0.12-NOTES.txt +++ b/docs/PUBLIC-1.0.12-NOTES.txt @@ -15,6 +15,8 @@ FOV slider so you can see it changing while you play. - Added remembered Discord sign-in on Linux and Steam Deck without a keyring. - Made the offline FOV slider work live, including while paused. - Fixed logging after an updater restart and improved join diagnostics. +- Fixed installed Store mods showing up in Needs Attention after a restart. + You won't need to download those installed mods again. All the existing memory, HUD, modded lobby and spectator fixes are kept. Your saves, mods and settings can stay. Please use the complete v1.0.12 diff --git a/patches/character-library-portraits/ReleaseOverlay.cmake b/patches/character-library-portraits/ReleaseOverlay.cmake index 52dce4ec..c83fe305 100644 --- a/patches/character-library-portraits/ReleaseOverlay.cmake +++ b/patches/character-library-portraits/ReleaseOverlay.cmake @@ -24,6 +24,7 @@ if(PROJECT_NAME STREQUAL "DKRRuntimePreparation") file(GLOB authored CONFIGURE_DEPENDS "${auth}/native/*" "${auth}/tests/*") file(GLOB profile_authored CONFIGURE_DEPENDS "${DKRPORT_ROOT}/runtime-recomp/src/game/online_profile/*") list(APPEND authored ${profile_authored}) + list(APPEND authored "${DKRPORT_ROOT}/runtime-recomp/src/game/mod_store/download_history.hpp") set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${authored} "${auth}/pipeline.py" "${auth}/ReleaseOverlay.cmake") execute_process(COMMAND "${Python3_EXECUTABLE}" -B "${auth}/pipeline.py" --inputs "${stage}/inputs.json" --output "${stage}" RESULT_VARIABLE result) if(NOT result EQUAL 0) @@ -75,6 +76,10 @@ if(PROJECT_NAME STREQUAL "DKRRuntimePreparation") add_executable(DKRProfileChecks EXCLUDE_FROM_ALL "${DKRPORT_ROOT}/runtime-recomp/tests/online_profile_tests.cpp") target_compile_features(DKRProfileChecks PRIVATE cxx_std_20) target_include_directories(DKRProfileChecks PRIVATE "${DKRPORT_ROOT}/runtime-recomp/src/game") + add_executable(DKRModStoreHistoryChecks EXCLUDE_FROM_ALL "${DKRPORT_ROOT}/runtime-recomp/tests/mod_store_history_tests.cpp") + target_compile_features(DKRModStoreHistoryChecks PRIVATE cxx_std_20) + target_include_directories(DKRModStoreHistoryChecks PRIVATE "${DKRPORT_ROOT}/runtime-recomp/src/game" + "${DKRPORT_ROOT}/extern/rt64/src/contrib") get_target_property(links DKRPortGame LINK_DIRECTORIES) if(links) target_link_directories(DKRCharacterPortraitChecks PRIVATE ${links}) diff --git a/patches/character-library-portraits/pipeline.py b/patches/character-library-portraits/pipeline.py index a1bd9814..624d943b 100644 --- a/patches/character-library-portraits/pipeline.py +++ b/patches/character-library-portraits/pipeline.py @@ -154,6 +154,34 @@ def friends_layout(text,out): return text[:grid_start]+grid+text[grid_end:] def account_service(text,out): + # Extend the retained service at its existing composition point. The history + # policy is normal first-party source, shared with the ROM-free host tests. + history=ROOT/'runtime-recomp/src/game/mod_store/download_history.hpp' + text='#include "'+history.as_posix()+'"\n'+text + begin=text.index(' Json rows = Json::array();',text.index(' void persist_downloads()')) + end=text.index('\n }',begin) + text=text[:begin]+''' Json saved; + { + std::lock_guard lock(mutex); + saved = dkr::store::history::save(state.downloads); + } + write_json(root / "downloads.json", saved);'''+text[end:] + text=once(text,' const auto saved = read_json(p.root / "downloads.json");', + ''' const auto saved = read_json(p.root / "downloads.json"); + const auto receipts = read_json(p.root / "installations.json"); + auto restored = dkr::store::history::restore(saved, receipts, [&](const std::string& path) { + return dkr::store::history::member_present(p.root.parent_path(), path); + });''') + begin=text.index(' if (saved.contains("jobs")') + end=text.index('\n });',begin) + text=text[:begin]+' s.downloads = std::move(restored);'+text[end:] + text=once(text,' s.downloads = std::move(restored);\n });', + ''' s.downloads = std::move(restored); + }); + if (saved.is_object() && saved.contains("jobs") && saved["jobs"].is_array() && + saved["jobs"].size() <= 64 && (!saved.contains("schema") || saved["schema"] == 1 || saved["schema"] == 2)) { + try { p.persist_downloads(); } catch (const std::exception& error) { p.fail(error); } + }''') text=once(text,'namespace dkr::store {', '#include "'+(out/'display_name_save.hpp').as_posix()+'"\n#include "'+(out/'profile_bridge.hpp').as_posix()+'"\n#include "'+(out/'picture_codec.hpp').as_posix()+'"\n#include "'+(out/'profile_retry.hpp').as_posix()+'"\n#define DKR_PROFILE_WITH_KEYRING 1\n#include "'+(out/'linux_device_store.hpp').as_posix()+'"\n\nnamespace dkr::store {') for operation in ('store','load','clear'): diff --git a/patches/release-prep/v1.0.11/build.py b/patches/release-prep/v1.0.11/build.py index bfcf6501..3bd95343 100644 --- a/patches/release-prep/v1.0.11/build.py +++ b/patches/release-prep/v1.0.11/build.py @@ -9,7 +9,7 @@ names=['DKRPortGame',*[v[0] for v in TARGETS.values()],'DKRVehicleOriginalChecks','DKRVehicleOriginalBeforeChecks'] names += [f'DKROnlineMenu{r}{s}Checks' for r in (77,80) for s in ('Before','')] names += ['DKROnlineSetupChecks','DKROnlinePartyChecks','DKRDirectoryLifecycleChecks', - 'DKRCharacterPortraitChecks','DKRProfileChecks'] + 'DKRCharacterPortraitChecks','DKRProfileChecks','DKRModStoreHistoryChecks'] if a.sanitizer:names.append('DKRVehicleOriginalSanitizerChecks') a.log.parent.mkdir(parents=True,exist_ok=True) with a.log.open('w') as out:subprocess.run([a.cmake,'--build',str(a.build),'--config','RelWithDebInfo','--target',*names,'--parallel',str(a.jobs)],stdout=out,stderr=subprocess.STDOUT,check=True) diff --git a/runtime-recomp/CMakeLists.txt b/runtime-recomp/CMakeLists.txt index 1c5c664d..955f1929 100644 --- a/runtime-recomp/CMakeLists.txt +++ b/runtime-recomp/CMakeLists.txt @@ -2637,6 +2637,12 @@ if(BUILD_TESTING) "${DKR_DEPENDENCIES_DIR}/n64-modern-runtime/N64Recomp/include") add_test(NAME DKRLiveFov COMMAND DKRLiveFovTests) + add_executable(DKRModStoreHistoryTests tests/mod_store_history_tests.cpp) + target_compile_features(DKRModStoreHistoryTests PRIVATE cxx_std_20) + target_include_directories(DKRModStoreHistoryTests PRIVATE "${CMAKE_CURRENT_LIST_DIR}/src/game" + "${DKRPORT_ROOT}/extern/rt64/src/contrib") + add_test(NAME DKRModStoreHistory COMMAND DKRModStoreHistoryTests) + add_test(NAME DKRTestAssertionContract COMMAND "${CMAKE_COMMAND}" "-DTEST_BINARY_DIR=${CMAKE_CURRENT_BINARY_DIR}/test-assertion-contract" diff --git a/runtime-recomp/src/game/mod_store/download_history.hpp b/runtime-recomp/src/game/mod_store/download_history.hpp new file mode 100644 index 00000000..50aefd4f --- /dev/null +++ b/runtime-recomp/src/game/mod_store/download_history.hpp @@ -0,0 +1,161 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace dkr::store::history { +inline constexpr std::uint64_t max_package_bytes = 2ULL * 1024 * 1024 * 1024; + +template std::string field(const Json& value, const char* name, std::size_t limit) { + if (!value.is_object()) + return {}; + const auto it = value.find(name); + if (it == value.end() || !it->is_string()) + return {}; + const auto& text = it->template get_ref(); + return text.size() <= limit ? text : std::string{}; +} + +inline bool identifier(std::string_view value) { + return !value.empty() && value.size() <= 160 && std::all_of(value.begin(), value.end(), [](unsigned char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' || c == '-'; + }); +} + +inline bool digest(std::string_view value) { + return value.size() == 64 && std::all_of(value.begin(), value.end(), [](char c) { + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'); + }); +} + +template std::uint64_t bytes(const Json& value) { + if (!value.is_object()) + return 0; + const auto it = value.find("bytes"); + if (it == value.end() || !it->is_number_unsigned()) + return 0; + const auto result = it->template get(); + return result <= max_package_bytes ? result : 0; +} + +// Only used by the background history loader. This never claims ownership, +// hashes whole packages, changes files or replaces native-library repair checks. +inline std::filesystem::path utf8_path(std::string_view value) { + return std::filesystem::path(std::u8string(value.begin(), value.end())); +} + +inline bool member_present(const std::filesystem::path& config, const std::string& relative) { + if (relative.empty() || relative.find_first_of("\\:") != relative.npos) + return false; + const auto path = utf8_path(relative); + if (path.is_absolute() || path.has_root_name()) + return false; + auto at = config; + std::error_code error; + for (const auto& part : path) { + if (part.empty() || part == "." || part == "..") + return false; + at /= part; + const auto status = std::filesystem::symlink_status(at, error); + if (error || std::filesystem::is_symlink(status) || + (!std::filesystem::is_directory(status) && !std::filesystem::is_regular_file(status))) + return false; + } + return true; +} + +template +const Json* completed_receipt(const Job& job, const Json& registry, Present&& present) { + if (!registry.is_object() || !registry.contains("schema") || registry["schema"] != 1 || + !registry.contains("records") || !registry["records"].is_array()) + return nullptr; + // Explicit replacement/update attempts may be paused even when an older + // successful install of that release exists. Leave those attempts alone. + if (!job.intent.empty() || !job.instance.empty()) + return nullptr; + for (const auto& record : registry["records"]) { + if (field(record, "item", 160) != job.item || field(record, "release", 160) != job.release || + field(record, "category", 32) != job.category || field(record, "state", 80) != "Installed" || + (record.contains("stored") && record["stored"] != false)) + continue; + const auto hash = field(record, "digest", 64); + if (!digest(hash) || (!job.digest.empty() && job.digest != hash) || !record.contains("members") || + !record["members"].is_array() || record["members"].empty()) + continue; + const bool complete = std::all_of(record["members"].begin(), record["members"].end(), [&](const Json& member) { + return !field(member, "key", 512).empty() && !field(member, "kind", 40).empty() && + present(field(member, "path", 4096)); + }); + if (complete) + return &record; + } + return nullptr; +} + +template +std::vector restore(const Json& saved, const Json& registry, Present&& present) { + std::vector result; + if (!saved.is_object() || !saved.contains("jobs") || !saved["jobs"].is_array() || saved["jobs"].size() > 64) + return result; + const bool legacy = !saved.contains("schema") || saved["schema"] == 1; + if (!legacy && saved["schema"] != 2) + return result; + for (const auto& row : saved["jobs"]) { + Job job; + job.item = field(row, "item", 160); + job.release = field(row, "release", 160); + job.title = field(row, "title", 200); + job.category = field(row, "category", 32); + job.version = field(row, "version", 80); + job.intent = field(row, "intent", 32); + job.instance = field(row, "instance", 512); + if (!identifier(job.item) || !identifier(job.release)) + continue; + job.key = job.item + "_" + job.release + (job.instance.empty() ? "" : "_" + job.instance); + if (std::any_of(result.begin(), result.end(), [&](const Job& old) { return old.key == job.key; })) + continue; + const auto hash = field(row, "sha256", 64); + if (digest(hash)) + job.digest = hash; + job.total = bytes(row); + const auto previous = field(row, "state", 80); + const Json* receipt = nullptr; + if (legacy && previous == "Paused") + receipt = completed_receipt(job, registry, present); + if (previous == "Installed" || receipt) { + job.state = "Installed"; + if (receipt) { + job.digest = field(*receipt, "digest", 64); + if (!job.total) + job.total = bytes(*receipt); + } + job.received = job.total; + } else { + // Resuming always obtains current integrity metadata and verifies + // the cache. No filename or queued/active state is trusted on boot. + job.state = previous == "Failed" || previous == "Install failed" ? "Failed" : "Paused"; + job.error = field(row, "error", 2000); + if (job.error.empty()) + job.error = previous == "Install failed" ? "Installation did not finish. Resume to verify the saved package." + : "Resume to check or reuse the saved download."; + } + result.push_back(std::move(job)); + } + return result; +} + +template Json save(const Jobs& jobs) { + Json rows = Json::array(); + for (const auto& job : jobs) + rows.push_back({{"item", job.item}, {"release", job.release}, {"title", job.title}, + {"category", job.category}, {"version", job.version}, {"state", job.state}, + {"sha256", job.digest}, {"bytes", job.total}, {"intent", job.intent}, + {"instance", job.instance}, {"error", job.error.size() <= 2000 ? job.error : std::string{}}}); + return {{"schema", 2}, {"jobs", std::move(rows)}}; +} +} // namespace dkr::store::history diff --git a/runtime-recomp/tests/mod_store_history_tests.cpp b/runtime-recomp/tests/mod_store_history_tests.cpp new file mode 100644 index 00000000..bc218b6b --- /dev/null +++ b/runtime-recomp/tests/mod_store_history_tests.cpp @@ -0,0 +1,160 @@ +#include "mod_store/download_history.hpp" +#include +#include +#include +#include +#include + +namespace { +using namespace dkr::store::history; +using Json = nlohmann::json; +struct Job { + std::string key, item, release, title, category, state, error, digest, version, intent, instance; + std::filesystem::path file; + std::uint64_t received = 0, total = 0; +}; +unsigned checks = 0; +void check(bool condition) { + ++checks; + if (!condition) + throw std::runtime_error("Mod Store history check " + std::to_string(checks) + " failed"); +} +const std::string hash(64, 'a'); +Json row(std::string state = "Installed") { + return {{"item", "mod_one"}, {"release", "release_one"}, {"category", "characters"}, + {"title", "Test character"}, {"state", state}, {"sha256", hash}, {"bytes", 100ULL}, + {"version", "1.0.0"}, {"intent", ""}, {"instance", ""}}; +} +Json receipt() { + return {{"schema", 1}, {"records", Json::array({ + {{"item", "mod_one"}, {"release", "release_one"}, {"category", "characters"}, + {"state", "Installed"}, {"stored", false}, {"digest", hash}, {"bytes", 100ULL}, + {"members", Json::array({{{"key", "character:one"}, {"kind", "legacy-character"}, + {"path", "mods/character-one"}}})}} + })}}; +} +Json saved(const Json& job, unsigned schema = 1) { + return {{"schema", schema}, {"jobs", Json::array({job})}}; +} +auto load(const Json& data, const Json& receipts = Json::object(), bool present = true) { + return restore(data, receipts, [present](const std::string& path) { return present && !path.empty(); }); +} +struct Temp { + std::filesystem::path path = std::filesystem::temp_directory_path() / + ("dkrr-history-" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + Temp() { std::filesystem::create_directories(path / "mods"); } + ~Temp() { std::error_code error; std::filesystem::remove_all(path, error); } +}; +} // namespace + +int main(int argc, char** argv) try { + auto jobs = load(saved(row())); + check(jobs.size() == 1 && jobs[0].state == "Installed" && jobs[0].error.empty()); + check(jobs[0].digest == hash && jobs[0].received == 100 && jobs[0].total == 100); + for (unsigned restart = 0; restart < 5; ++restart) { + jobs = load(Json::parse(save(jobs).dump())); + check(jobs.size() == 1 && jobs[0].state == "Installed" && jobs[0].digest == hash); + } + auto legacy = row("Paused"); + legacy["sha256"] = ""; + legacy["bytes"] = 0ULL; + jobs = load(saved(legacy), receipt()); + check(jobs.size() == 1 && jobs[0].state == "Installed" && jobs[0].digest == hash && jobs[0].total == 100); + check(load(saved(legacy), receipt(), false)[0].state == "Paused"); + check(load(saved(legacy), Json::object())[0].state == "Paused"); + check(load(saved(legacy, 2), receipt())[0].state == "Paused"); + for (const auto* field : {"item", "release", "category", "digest", "state"}) { + auto r = receipt(); + r["records"][0][field] = "different"; + check(load(saved(legacy), r)[0].state == "Paused"); + } + for (const auto* state : {"Missing", "Needs repair", "Stored edition"}) { + auto r = receipt(); + r["records"][0]["state"] = state; + check(load(saved(legacy), r)[0].state == "Paused"); + } + auto r = receipt(); + r["records"][0]["stored"] = true; + check(load(saved(legacy), r)[0].state == "Paused"); + r = receipt(); + r["records"][0]["members"] = Json::array(); + check(load(saved(legacy), r)[0].state == "Paused"); + r = receipt(); + r["records"][0]["members"].push_back({{"key", "second"}, {"kind", "legacy-character"}, {"path", ""}}); + check(load(saved(legacy), r)[0].state == "Paused"); + for (const auto* field : {"intent", "instance"}) { + auto pending = legacy; + pending[field] = "explicit-operation"; + check(load(saved(pending), receipt())[0].state == "Paused"); + } + auto different_hash = legacy; + different_hash["sha256"] = std::string(64, 'b'); + check(load(saved(different_hash), receipt())[0].state == "Paused"); + for (const auto* state : {"Queued", "Downloading", "Verifying", "Installing", "Preparing download", + "Ready to install", "Paused for gameplay", "Paused"}) { + jobs = load(saved(row(state), 2), receipt()); + check(jobs[0].state == "Paused" && jobs[0].file.empty() && !jobs[0].error.empty()); + } + for (const auto* state : {"Failed", "Install failed"}) { + auto failed = row(state); + failed["error"] = "The package could not be verified."; + jobs = load(saved(failed), receipt()); + check(jobs[0].state == "Failed" && jobs[0].error == failed["error"].get()); + check(load(Json::parse(save(jobs).dump()), receipt())[0].error == failed["error"].get()); + } + for (const auto& malformed : {Json(), Json::array(), Json("invalid"), Json{{"jobs", "invalid"}}, + saved(row(), 99)}) + check(load(malformed).empty()); + for (const auto* field : {"item", "release"}) { + auto invalid = row(); + invalid[field] = "../outside"; + check(load(saved(invalid)).empty()); + } + auto mixed = saved(row()); + mixed["jobs"].push_back(nullptr); + mixed["jobs"].push_back(row()); + check(load(mixed).size() == 1); + for (const auto& size : {Json(-1), Json("100"), Json(max_package_bytes + 1)}) { + auto invalid = row(); + invalid["bytes"] = size; + check(load(saved(invalid))[0].total == 0); + } + Temp temp; + std::ofstream(temp.path / "mods/character-one") << "fixture"; + check(member_present(temp.path, "mods/character-one")); + check(member_present(temp.path, "mods")); + for (const auto* path : {"", "../outside", "/outside", "C:/outside", "mods/../character-one", + "mods\\character-one", "mods/./character-one", "mods/missing"}) + check(!member_present(temp.path, path)); + std::error_code error; + std::filesystem::create_symlink(temp.path / "mods/character-one", temp.path / "mods/link", error); + if (!error) + check(!member_present(temp.path, "mods/link")); + jobs = restore(saved(legacy), receipt(), [&](const std::string& path) { return member_present(temp.path, path); }); + check(jobs[0].state == "Installed"); + std::filesystem::remove(temp.path / "mods/character-one"); + jobs = restore(saved(legacy), receipt(), [&](const std::string& path) { return member_present(temp.path, path); }); + check(jobs[0].state == "Paused"); + std::cout << checks << " Mod Store history checks passed; no network or user profile used.\n"; + if (argc == 4) { + // Optional read-only inspection of copied records. Never save the + // migrated result or open a launcher, account or network connection. + std::ifstream history_file(utf8_path(argv[1])), receipt_file(utf8_path(argv[2])); + const auto data = Json::parse(history_file), receipts = Json::parse(receipt_file); + const auto config = utf8_path(argv[3]); + const auto restored = restore(data, receipts, [&](const std::string& path) { + return member_present(config, path); + }); + Json counts = Json::object(); + for (const auto& job : restored) { + if (!counts.contains(job.state)) + counts[job.state] = 0; + counts[job.state] = counts[job.state].get() + 1; + } + std::cout << "Read-only history inspection: " << counts.dump() << '\n'; + } + return 0; +} catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; +}