diff --git a/.github/workflows/model_validation.yml b/.github/workflows/model_validation.yml index 33234f2f..0ff9378c 100644 --- a/.github/workflows/model_validation.yml +++ b/.github/workflows/model_validation.yml @@ -28,7 +28,8 @@ env: jobs: validate-remote-models: runs-on: ubuntu-22.04 - timeout-minutes: 90 + # Worst case is ~14 models x (timeout + retry delay + retry) plus a cold build + timeout-minutes: 150 steps: - name: Install JUCE Linux dependencies @@ -52,8 +53,19 @@ jobs: with: submodules: recursive + - name: ccache + uses: hendrikmuhs/ccache-action@main + with: + key: v1-model-validation + - name: Configure - run: cmake -B "$BUILD_DIR" -G Ninja -DCMAKE_BUILD_TYPE="$BUILD_TYPE" . + run: > + cmake -B "$BUILD_DIR" -G Ninja + -DCMAKE_BUILD_TYPE="$BUILD_TYPE" + -DHARP_BUILD_MODEL_VALIDATION=ON + -DCMAKE_C_COMPILER_LAUNCHER=ccache + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + . - name: Build model validation runner run: cmake --build "$BUILD_DIR" --config "$BUILD_TYPE" --target HARPRemoteModelTests --parallel 4 @@ -68,4 +80,4 @@ jobs: uses: actions/upload-artifact@v4 with: name: remote-model-validation-report - path: artifacts/model_validation/remote + path: ${{ env.HARP_MODEL_VALIDATION_REPORT_DIR }} diff --git a/CMakeLists.txt b/CMakeLists.txt index e08b9462..30473ec3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -169,42 +169,68 @@ if(WIN32) ) endif() -include(CTest) +# Validate the bundled model registry at configure time so a bad edit fails the +# build instead of silently degrading the in-app model list. +file(READ "${CMAKE_SOURCE_DIR}/resources/models/model_registry.json" HARP_MODEL_REGISTRY_CONTENTS) +string(JSON HARP_MODEL_REGISTRY_MODEL_COUNT + ERROR_VARIABLE HARP_MODEL_REGISTRY_ERROR + LENGTH "${HARP_MODEL_REGISTRY_CONTENTS}" models) + +if(HARP_MODEL_REGISTRY_ERROR) + message(FATAL_ERROR "resources/models/model_registry.json is invalid: ${HARP_MODEL_REGISTRY_ERROR}") +elseif(HARP_MODEL_REGISTRY_MODEL_COUNT LESS 1) + message(FATAL_ERROR "resources/models/model_registry.json contains no models.") +endif() -juce_add_console_app(HARPRemoteModelTests - PRODUCT_NAME "HARPRemoteModelTests" -) +# The validation runner performs live network calls against deployed models, so +# it is opt-in rather than part of the default build and test suite. +option(HARP_BUILD_MODEL_VALIDATION "Build the remote model validation runner (live-network tests)" OFF) -juce_generate_juce_header(HARPRemoteModelTests) +if(HARP_BUILD_MODEL_VALIDATION) + include(CTest) -target_sources(HARPRemoteModelTests - PRIVATE - tests/ModelValidationMain.cpp - ${HARP_COMMON_SOURCES} -) + juce_add_console_app(HARPRemoteModelTests + PRODUCT_NAME "HARPRemoteModelTests" + ) -target_include_directories(HARPRemoteModelTests - PRIVATE - "${CMAKE_SOURCE_DIR}/src" -) + juce_generate_juce_header(HARPRemoteModelTests) -target_compile_definitions(HARPRemoteModelTests - PRIVATE - JUCE_USE_CURL=1 - JUCE_LOAD_CURL_SYMBOLS_LAZILY=1 - HARP_SOURCE_DIR="${CMAKE_SOURCE_DIR}" -) + target_sources(HARPRemoteModelTests + PRIVATE + tests/ModelValidationMain.cpp + ${HARP_COMMON_SOURCES} + ) -target_link_libraries(HARPRemoteModelTests - PRIVATE - ${HARP_JUCE_MODULES} - stability_controls - PUBLIC - ${HARP_JUCE_FLAGS} -) + target_include_directories(HARPRemoteModelTests + PRIVATE + "${CMAKE_SOURCE_DIR}/src" + ) + + target_compile_definitions(HARPRemoteModelTests + PRIVATE + JUCE_WEB_BROWSER=0 + JUCE_USE_CURL=1 + JUCE_LOAD_CURL_SYMBOLS_LAZILY=1 + JUCE_USE_OGGVORBIS=1 + JUCE_USE_MP3AUDIOFORMAT=1 + HARP_SOURCE_DIR="${CMAKE_SOURCE_DIR}" + ) + + target_link_libraries(HARPRemoteModelTests + PRIVATE + ${HARP_JUCE_MODULES} + stability_controls + PUBLIC + ${HARP_JUCE_FLAGS} + ) + + if(BUILD_TESTING) + add_test(NAME HARPRemoteModelTests COMMAND HARPRemoteModelTests) -if(BUILD_TESTING) - add_test(NAME HARPRemoteModelTests COMMAND HARPRemoteModelTests) + # The serial sweep can legitimately run for over an hour; ctest's default + # 1500s timeout would kill it mid-run before any report is written. + set_tests_properties(HARPRemoteModelTests PROPERTIES TIMEOUT 7200) + endif() endif() if(APPLE) diff --git a/resources/media/test.wav b/resources/media/test.wav index c1af7c93..6ac2cd4d 100644 Binary files a/resources/media/test.wav and b/resources/media/test.wav differ diff --git a/resources/media/test_old.wav b/resources/media/test_old.wav deleted file mode 100644 index 6ac2cd4d..00000000 Binary files a/resources/media/test_old.wav and /dev/null differ diff --git a/resources/models/model_registry.json b/resources/models/model_registry.json index 4c9d4c21..caa48055 100644 --- a/resources/models/model_registry.json +++ b/resources/models/model_registry.json @@ -1,30 +1,14 @@ { "version": 1, "models": [ - { - "id": "custom-path", - "name": "Custom Path", - "path": "click here to enter a custom path...", - "provider": "custom", - "source": "ui", - "featured": true, - "validation": { - "enabled": false, - "mode": "manual", - "reason": "UI placeholder entry used to add custom model paths." - } - }, { "id": "stability-text-to-audio", "name": "Stable Audio Text to Audio", "path": "stability/text-to-audio", - "provider": "stability", - "source": "stability_api", "featured": true, "validation": { "enabled": true, "mode": "remote_api", - "requires_network": true, "requires_env": "HARP_STABILITY_API_KEY" } }, @@ -32,13 +16,10 @@ "id": "stability-audio-to-audio", "name": "Stable Audio Audio to Audio", "path": "stability/audio-to-audio", - "provider": "stability", - "source": "stability_api", "featured": true, "validation": { "enabled": true, "mode": "remote_api", - "requires_network": true, "requires_env": "HARP_STABILITY_API_KEY" } }, @@ -46,240 +27,121 @@ "id": "text2midi-symbolic-music-generation", "name": "Text2MIDI Symbolic Music Generation", "path": "teamup-tech/text2midi-symbolic-music-generation", - "provider": "huggingface", - "source": "huggingface_space", "featured": true, "validation": { "enabled": true, - "mode": "remote_gradio", - "requires_network": true + "mode": "remote_gradio" } }, { "id": "demucs-source-separation", "name": "Demucs Source Separation", "path": "teamup-tech/demucs-source-separation", - "provider": "huggingface", - "source": "huggingface_space", "featured": true, "validation": { "enabled": true, - "mode": "remote_gradio", - "requires_network": true + "mode": "remote_gradio" } }, { "id": "solo-piano-audio-to-midi-transcription", "name": "Solo Piano Audio to MIDI Transcription", "path": "teamup-tech/solo-piano-audio-to-midi-transcription", - "provider": "huggingface", - "source": "huggingface_space", "featured": true, "validation": { "enabled": true, - "mode": "remote_gradio", - "requires_network": true + "mode": "remote_gradio" } }, { "id": "transkun", "name": "Transkun", "path": "teamup-tech/transkun", - "provider": "huggingface", - "source": "huggingface_space", "featured": true, "validation": { "enabled": true, - "mode": "remote_gradio", - "requires_network": true + "mode": "remote_gradio" } }, { "id": "tria", - "name": "TRIA", + "name": "TRIA (The Rhythm In Anything)", "path": "teamup-tech/TRIA", - "provider": "huggingface", - "source": "huggingface_space", "featured": true, "validation": { "enabled": true, - "mode": "remote_gradio", - "requires_network": true + "mode": "remote_gradio" } }, { "id": "anticipatory-music-transformer", "name": "Anticipatory Music Transformer", "path": "teamup-tech/anticipatory-music-transformer", - "provider": "huggingface", - "source": "huggingface_space", "featured": true, "validation": { "enabled": true, - "mode": "remote_gradio", - "requires_network": true + "mode": "remote_gradio" } }, { "id": "vampnet-conditional-music-generation", "name": "VampNet Conditional Music Generation", "path": "teamup-tech/vampnet-conditional-music-generation", - "provider": "huggingface", - "source": "huggingface_space", "featured": true, "validation": { "enabled": true, - "mode": "remote_gradio", - "requires_network": true + "mode": "remote_gradio" } }, { "id": "harmonic-percussive-separation", "name": "Harmonic Percussive Separation", "path": "teamup-tech/harmonic-percussive-separation", - "provider": "huggingface", - "source": "huggingface_space", "featured": true, "validation": { "enabled": true, - "mode": "remote_gradio", - "requires_network": true + "mode": "remote_gradio" } }, { "id": "kokoro-tts", "name": "Kokoro TTS", "path": "teamup-tech/Kokoro-TTS", - "provider": "huggingface", - "source": "huggingface_space", "featured": true, "validation": { "enabled": true, - "mode": "remote_gradio", - "requires_network": true + "mode": "remote_gradio" } }, { "id": "megatts3-voice-cloning", "name": "MegaTTS3 Voice Cloning", "path": "teamup-tech/MegaTTS3-Voice-Cloning", - "provider": "huggingface", - "source": "huggingface_space", "featured": true, "validation": { "enabled": true, - "mode": "remote_gradio", - "requires_network": true + "mode": "remote_gradio" } }, { "id": "midi-synthesizer", "name": "MIDI Synthesizer", "path": "teamup-tech/midi-synthesizer", - "provider": "huggingface", - "source": "huggingface_space", "featured": true, "validation": { "enabled": true, - "mode": "remote_gradio", - "requires_network": true + "mode": "remote_gradio" } }, { "id": "audioseal", "name": "AudioSeal", "path": "teamup-tech/audioseal", - "provider": "huggingface", - "source": "huggingface_space", "featured": true, "validation": { "enabled": true, - "mode": "remote_gradio", - "requires_network": true - } - }, - { - "id": "pitch-shifter-example", - "name": "Pitch Shifter Example", - "path": null, - "provider": "pyharp", - "source": "local_example", - "featured": false, - "validation": { - "enabled": true, - "mode": "local_pyharp_example", - "app_path": "pyharp/examples/pitch_shifter/app.py", - "python_modules": [ - "gradio", - "audiotools", - "torch", - "torchaudio" - ], - "inputs": [ - { - "kind": "audio_fixture", - "path": "resources/media/test.wav" - }, - { - "kind": "literal", - "value": 7 - } - ], - "expected_outputs": [ - { - "kind": "audio_file", - "extension": ".wav", - "min_size_bytes": 128 - } - ] - } - }, - { - "id": "midi-pitch-shifter-example", - "name": "MIDI Pitch Shifter Example", - "path": null, - "provider": "pyharp", - "source": "local_example", - "featured": false, - "validation": { - "enabled": true, - "mode": "local_pyharp_example", - "app_path": "pyharp/examples/midi_pitch_shifter/app.py", - "python_modules": [ - "gradio", - "symusic" - ], - "inputs": [ - { - "kind": "midi_fixture", - "path": "resources/media/test.mid" - }, - { - "kind": "literal", - "value": 7 - } - ], - "expected_outputs": [ - { - "kind": "midi_file", - "extension": ".mid", - "min_size_bytes": 32 - } - ] - } - }, - { - "id": "ui-tester-example", - "name": "UI Tester Example", - "path": null, - "provider": "pyharp", - "source": "local_example", - "featured": false, - "validation": { - "enabled": false, - "mode": "local_pyharp_example", - "reason": "Downloads external fixture files and is better covered by a dedicated integration test." + "mode": "remote_gradio" } } ] -} \ No newline at end of file +} diff --git a/src/clients/GradioClient.h b/src/clients/GradioClient.h index 9b77390f..d87bf5e3 100644 --- a/src/clients/GradioClient.h +++ b/src/clients/GradioClient.h @@ -692,7 +692,8 @@ class GradioClient : public Client // TODO - could potentially identify other errors (e.g., too many requests) - return OpResult::fail(GradioError { GradioError::Type::RuntimeError, errorPath }); + return OpResult::fail( + GradioError { GradioError::Type::RuntimeError, errorPath, response }); } else { diff --git a/src/utils/Errors.h b/src/utils/Errors.h index fd9c8318..8c94d1a6 100644 --- a/src/utils/Errors.h +++ b/src/utils/Errors.h @@ -227,6 +227,8 @@ struct GradioError Type type; String endpointPath; + + String detail; // Raw error payload reported by the remote endpoint, if any }; inline String toUserMessage(const GradioError& e) @@ -247,6 +249,11 @@ inline String toUserMessage(const GradioError& e) userMessage += ". If this is a Hugging Face space running on ZeroGPU, this " "can also indicate a user has exceeded their daily ZeroGPU quota."; + if (e.detail.isNotEmpty()) + { + userMessage += " Remote error: \"" + e.detail + "\"."; + } + return userMessage; } diff --git a/src/utils/ModelRegistry.h b/src/utils/ModelRegistry.h index bc4bb423..56ab6144 100644 --- a/src/utils/ModelRegistry.h +++ b/src/utils/ModelRegistry.h @@ -5,6 +5,7 @@ #pragma once +#include #include #include @@ -16,61 +17,42 @@ using namespace juce; namespace ModelRegistry { -inline std::vector getFallbackFeaturedModelPaths() -{ - return { - "click here to enter a custom path...", - "stability/text-to-audio", - "stability/audio-to-audio", - "teamup-tech/text2midi-symbolic-music-generation", - "teamup-tech/demucs-source-separation", - "teamup-tech/solo-piano-audio-to-midi-transcription", - "teamup-tech/transkun", // TODO - more intuitive name - "teamup-tech/TRIA", // TODO - more intuitive name: (The Rhythm In Anything) conditional drum generation - "teamup-tech/anticipatory-music-transformer", - "teamup-tech/vampnet-conditional-music-generation", - "teamup-tech/harmonic-percussive-separation", - "teamup-tech/Kokoro-TTS", - "teamup-tech/MegaTTS3-Voice-Cloning", - "teamup-tech/midi-synthesizer", - "teamup-tech/audioseal", // TODO - more intuitive name - // "xribene/HARP-UI-TEST-v3" - }; -} +// The model selection UI treats the first combo-box entry as the "enter a custom +// path" affordance, so the placeholder is prepended here rather than stored in +// the registry, keeping the registry models-only. +inline constexpr const char* customPathPlaceholder = "click here to enter a custom path..."; inline std::vector getFeaturedModelPaths() { + std::vector featuredPaths { customPathPlaceholder }; + const String registryJson = String::fromUTF8(BinaryData::model_registry_json, BinaryData::model_registry_jsonSize); var parsedRegistry; const Result parseResult = JSON::parse(registryJson, parsedRegistry); + // The bundled registry is validated at configure time (see CMakeLists.txt), + // so failures here indicate a corrupted build rather than a bad edit. if (parseResult.failed() || ! parsedRegistry.isObject()) { - DBG_AND_LOG("ModelRegistry::getFeaturedModelPaths: Failed to parse bundled registry. " - "Using fallback list."); + DBG_AND_LOG("ModelRegistry::getFeaturedModelPaths: Failed to parse bundled registry."); - return getFallbackFeaturedModelPaths(); + return featuredPaths; } auto* root = parsedRegistry.getDynamicObject(); - if (root == nullptr || ! root->hasProperty("models") || ! root->getProperty("models").isArray()) + if (root == nullptr || ! root->getProperty("models").isArray()) { DBG_AND_LOG("ModelRegistry::getFeaturedModelPaths: Registry is missing a valid models " - "array. Using fallback list."); + "array."); - return getFallbackFeaturedModelPaths(); + return featuredPaths; } - std::vector featuredPaths; - for (const auto& modelVar : *root->getProperty("models").getArray()) { - if (! modelVar.isObject()) - continue; - auto* model = modelVar.getDynamicObject(); if (model == nullptr) @@ -83,14 +65,6 @@ inline std::vector getFeaturedModelPaths() featuredPaths.push_back(path.toStdString()); } - if (featuredPaths.empty()) - { - DBG_AND_LOG("ModelRegistry::getFeaturedModelPaths: Registry did not yield any featured " - "paths. Using fallback list."); - - return getFallbackFeaturedModelPaths(); - } - return featuredPaths; } -} // namespace ModelRegistry \ No newline at end of file +} // namespace ModelRegistry diff --git a/src/widgets/ModelSelectionWidget.h b/src/widgets/ModelSelectionWidget.h index 900c5fc8..26c81a38 100644 --- a/src/widgets/ModelSelectionWidget.h +++ b/src/widgets/ModelSelectionWidget.h @@ -539,4 +539,4 @@ class ModelSelectionWidget : public Component, public ChangeBroadcaster, public MultiButton::Mode loadButtonActiveInfo; SharedResourcePointer instructionsMessage; -}; \ No newline at end of file +}; diff --git a/tests/ModelValidationMain.cpp b/tests/ModelValidationMain.cpp index 6986ffde..a6dde25d 100644 --- a/tests/ModelValidationMain.cpp +++ b/tests/ModelValidationMain.cpp @@ -1,11 +1,15 @@ #include -#include #include +#include +#include #include +#include +#include #include #include "Model.h" +#include "utils/Errors.h" #include "utils/Logging.h" JUCE_IMPLEMENT_SINGLETON(HARPLogger) @@ -17,7 +21,10 @@ namespace constexpr auto registryRelativePath = "resources/models/model_registry.json"; constexpr auto audioFixtureRelativePath = "resources/media/test.wav"; constexpr auto midiFixtureRelativePath = "resources/media/test.mid"; +constexpr auto defaultTextControlValue = "Short validation prompt"; constexpr int defaultPerModelTimeoutMs = 120000; +constexpr int defaultMaxRetries = 0; +constexpr int defaultRetryDelayMs = 15000; struct ValidationEntry { @@ -26,6 +33,7 @@ struct ValidationEntry String path; String mode; String requiredEnv; + String prompt; }; struct ValidationResultRow @@ -33,10 +41,38 @@ struct ValidationResultRow String id; String name; String path; - String outcome; + String outcome; // "passed" | "failed" | "skipped" | "inconclusive" String reason; + bool retryable = false; +}; + +struct Summary +{ + int passed = 0; + int failed = 0; + int skipped = 0; + int inconclusive = 0; }; +Summary summarize(const std::vector& results) +{ + Summary summary; + + for (const auto& result : results) + { + if (result.outcome == "passed") + summary.passed += 1; + else if (result.outcome == "skipped") + summary.skipped += 1; + else if (result.outcome == "inconclusive") + summary.inconclusive += 1; + else + summary.failed += 1; + } + + return summary; +} + File repoRoot() { return File(HARP_SOURCE_DIR); @@ -72,11 +108,6 @@ String getSelectedModelId() return getEnvValue("HARP_MODEL_VALIDATION_ID"); } -bool isChildMode() -{ - return getEnvValue("HARP_MODEL_VALIDATION_CHILD") == "1"; -} - int getPerModelTimeoutMs() { const auto value = getEnvValue("HARP_MODEL_VALIDATION_TIMEOUT_MS"); @@ -90,9 +121,6 @@ int getPerModelTimeoutMs() return parsed > 0 ? parsed : defaultPerModelTimeoutMs; } -constexpr int defaultMaxRetries = 0; -constexpr int defaultRetryDelayMs = 15000; - int getMaxRetries() { const auto value = getEnvValue("HARP_MODEL_VALIDATION_RETRIES"); @@ -131,17 +159,15 @@ File getReportDir() return repoRoot().getChildFile("artifacts/model_validation/remote"); } -void seedProviderTokens(const ValidationEntry& entry) +void seedProviderTokens(const ValidationEntry& entry, SharedAPIKeys& sharedTokens) { - SharedResourcePointer sharedTokens; - if (entry.mode == "remote_api") { const auto token = firstNonEmptyEnv({ "HARP_STABILITY_API_KEY", "STABILITY_API_KEY" }); if (token.isNotEmpty()) { - sharedTokens->savedTokens[Provider::Stability] = token; + sharedTokens.savedTokens[Provider::Stability] = token; } } else @@ -150,7 +176,7 @@ void seedProviderTokens(const ValidationEntry& entry) if (token.isNotEmpty()) { - sharedTokens->savedTokens[Provider::HuggingFace] = token; + sharedTokens.savedTokens[Provider::HuggingFace] = token; } } } @@ -167,7 +193,7 @@ var parseJsonFile(const File& file) return parsed; } -std::vector loadRemoteValidationEntries() +std::vector loadRemoteValidationEntries(const String& selectedModelId) { const auto parsedRegistry = parseJsonFile(repoRoot().getChildFile(registryRelativePath)); @@ -183,7 +209,6 @@ std::vector loadRemoteValidationEntries() throw std::runtime_error("Model registry must contain a models array."); } - const auto selectedModelId = getSelectedModelId(); std::vector entries; std::unordered_set seenIds; @@ -228,13 +253,14 @@ std::vector loadRemoteValidationEntries() model->getProperty("path").toString(), mode, validation->getProperty("requires_env").toString(), + validation->getProperty("prompt").toString(), }); } return entries; } -void seedModelInputs(Model& model) +void seedModelInputs(Model& model, const ValidationEntry& entry) { const auto audioFixture = repoRoot().getChildFile(audioFixtureRelativePath); const auto midiFixture = repoRoot().getChildFile(midiFixtureRelativePath); @@ -257,13 +283,17 @@ void seedModelInputs(Model& model) } } + const auto textControlValue = + entry.prompt.isNotEmpty() ? entry.prompt.toStdString() + : std::string(defaultTextControlValue); + for (const auto& control : model.getControls()) { if (auto* text = dynamic_cast(control.get())) { if (text->value.empty()) { - text->value = "Short validation prompt"; + text->value = textControlValue; } } } @@ -271,29 +301,50 @@ void seedModelInputs(Model& model) String validateOutputFiles(const std::vector& outputFiles, const LabelList& labels) { + AudioFormatManager formatManager; + formatManager.registerBasicFormats(); + for (const auto& outputFile : outputFiles) { if (! outputFile.existsAsFile()) { - return "missing output file"; + return "missing output file " + outputFile.getFileName(); } const auto extension = outputFile.getFileExtension().toLowerCase(); if (extension == ".mid" || extension == ".midi") { - std::ifstream midiStream(outputFile.getFullPathName().toStdString(), std::ios::binary); - char header[4] {}; - midiStream.read(header, 4); + FileInputStream midiStream(outputFile); + MidiFile midiFile; + + if (! midiStream.openedOk() || ! midiFile.readFrom(midiStream)) + { + return "unreadable MIDI output " + outputFile.getFileName(); + } + + if (midiFile.getNumTracks() < 1) + { + return "MIDI output has no tracks: " + outputFile.getFileName(); + } + } + else if (formatManager.findFormatForFileExtension(extension) != nullptr) + { + std::unique_ptr reader(formatManager.createReaderFor(outputFile)); - if (String::fromUTF8(header, 4) != "MThd") + if (reader == nullptr) { - return "invalid MIDI output"; + return "undecodable audio output " + outputFile.getFileName(); + } + + if (reader->lengthInSamples <= 0) + { + return "empty audio output " + outputFile.getFileName(); } } else if (outputFile.getSize() <= 0) { - return "empty output file"; + return "empty output file " + outputFile.getFileName(); } } @@ -308,18 +359,49 @@ String validateOutputFiles(const std::vector& outputFiles, const LabelList return {}; } -String classifyFailureMessage(const String& message) +struct FailureInfo { - if (message.containsIgnoreCase("status code 503")) - return "503 Service Unavailable"; - if (message.containsIgnoreCase("timed out")) - return "remote Gradio timeout"; - if (message.containsIgnoreCase("runtime error occurred at endpoint")) - return "remote Gradio runtime error"; - if (message.containsIgnoreCase("valid API key")) - return "missing or invalid API key"; + String reason; + bool retryable = false; + bool inconclusive = false; +}; + +FailureInfo classifyFailure(const Error& error) +{ + FailureInfo info; + info.reason = toUserMessage(error); + + if (const auto* http = std::get_if(&error)) + { + // Sleeping/restarting spaces and server-side errors are transient + info.retryable = http->type == HttpError::Type::ConnectionFailed + || (http->type == HttpError::Type::BadStatusCode + && (http->statusCode == 429 || http->statusCode >= 500)); + } + else if (const auto* gradio = std::get_if(&error)) + { + // Quota exhaustion says nothing about the health of the model itself + info.inconclusive = gradio->detail.containsIgnoreCase("quota"); + } + else if (const auto* file = std::get_if(&error)) + { + info.retryable = file->type == FileError::Type::UploadFailed + || file->type == FileError::Type::DownloadFailed; + } + + return info; +} - return message.isNotEmpty() ? message : "unknown failure"; +ValidationResultRow makeFailureRow(const ValidationEntry& entry, const Error& error) +{ + const auto failure = classifyFailure(error); + + return { entry.id, + entry.name, + entry.path, + failure.inconclusive ? "inconclusive" : "failed", + failure.reason, + failure.retryable }; } String summarizeReason(const String& reason) @@ -339,28 +421,16 @@ String summarizeReason(const String& reason) return {}; } -String renderMarkdownReport(const std::vector& results) +String renderMarkdownReport(const std::vector& results, + const Summary& summary) { - int passed = 0; - int failed = 0; - int skipped = 0; - - for (const auto& result : results) - { - if (result.outcome == "passed") - passed += 1; - else if (result.outcome == "failed") - failed += 1; - else if (result.outcome == "skipped") - skipped += 1; - } - String markdown; markdown << "# HARP Remote Model Validation\n\n"; markdown << "- Total: " << results.size() << "\n"; - markdown << "- Passed: " << passed << "\n"; - markdown << "- Failed: " << failed << "\n"; - markdown << "- Skipped: " << skipped << "\n\n"; + markdown << "- Passed: " << summary.passed << "\n"; + markdown << "- Failed: " << summary.failed << "\n"; + markdown << "- Skipped: " << summary.skipped << "\n"; + markdown << "- Inconclusive: " << summary.inconclusive << "\n\n"; markdown << "## Dashboard\n\n"; markdown << "| Model Path | Outcome | Detail |\n"; markdown << "| --- | --- | --- |\n"; @@ -378,20 +448,11 @@ void writeReport(const File& reportDir, const std::vector& { reportDir.createDirectory(); - int passed = 0; - int failed = 0; - int skipped = 0; + const auto summary = summarize(results); Array rows; for (const auto& result : results) { - if (result.outcome == "passed") - passed += 1; - else if (result.outcome == "failed") - failed += 1; - else if (result.outcome == "skipped") - skipped += 1; - auto* row = new DynamicObject(); row->setProperty("id", result.id); row->setProperty("name", result.name); @@ -401,25 +462,22 @@ void writeReport(const File& reportDir, const std::vector& rows.add(var(row)); } - auto* summary = new DynamicObject(); - summary->setProperty("total", static_cast(results.size())); - summary->setProperty("passed", passed); - summary->setProperty("failed", failed); - summary->setProperty("skipped", skipped); + auto* summaryObject = new DynamicObject(); + summaryObject->setProperty("total", static_cast(results.size())); + summaryObject->setProperty("passed", summary.passed); + summaryObject->setProperty("failed", summary.failed); + summaryObject->setProperty("skipped", summary.skipped); + summaryObject->setProperty("inconclusive", summary.inconclusive); auto* report = new DynamicObject(); report->setProperty("generated_at", Time::getCurrentTime().toISO8601(true)); - report->setProperty("registry_path", repoRoot().getChildFile(registryRelativePath).getFullPathName()); - report->setProperty("summary", var(summary)); + report->setProperty("registry_path", + repoRoot().getChildFile(registryRelativePath).getFullPathName()); + report->setProperty("summary", var(summaryObject)); report->setProperty("results", rows); - const auto reportJson = JSON::toString(var(report), true); - const auto reportMd = renderMarkdownReport(results); - - reportDir.getChildFile("latest.json").replaceWithText(reportJson); - reportDir.getChildFile("latest.md").replaceWithText(reportMd); - reportDir.getChildFile("status.json").replaceWithText(reportJson); - reportDir.getChildFile("dashboard.md").replaceWithText(reportMd); + reportDir.getChildFile("latest.json").replaceWithText(JSON::toString(var(report), true)); + reportDir.getChildFile("latest.md").replaceWithText(renderMarkdownReport(results, summary)); } ValidationResultRow validateEntry(const ValidationEntry& entry) @@ -430,21 +488,21 @@ ValidationResultRow validateEntry(const ValidationEntry& entry) "Required environment variable " + entry.requiredEnv + " is not set." }; } - seedProviderTokens(entry); + // Must outlive the Model below — the shared object is destroyed with its + // last SharedResourcePointer, and the Client reading the tokens is only + // created inside Model::load + SharedResourcePointer sharedTokens; + seedProviderTokens(entry, *sharedTokens); Model model; const auto loadResult = model.load(entry.path); if (loadResult.failed()) { - return { entry.id, - entry.name, - entry.path, - "failed", - classifyFailureMessage(toUserMessage(loadResult.getError())) }; + return makeFailureRow(entry, loadResult.getError()); } - seedModelInputs(model); + seedModelInputs(model, entry); std::map inputFiles; @@ -472,19 +530,21 @@ ValidationResultRow validateEntry(const ValidationEntry& entry) if (processResult.failed()) { - return { entry.id, - entry.name, - entry.path, - "failed", - classifyFailureMessage(toUserMessage(processResult.getError())) }; + return makeFailureRow(entry, processResult.getError()); } - if (static_cast(outputFiles.size()) != static_cast(model.getOutputTracks().size())) + String outputError; + + if (outputFiles.size() != model.getOutputTracks().size()) { - return { entry.id, entry.name, entry.path, "failed", "unexpected number of output files" }; + outputError = "unexpected number of output files (got " + + String(outputFiles.size()) + ", expected " + + String(model.getOutputTracks().size()) + ")"; + } + else + { + outputError = validateOutputFiles(outputFiles, labels); } - - const auto outputError = validateOutputFiles(outputFiles, labels); for (const auto& outputFile : outputFiles) { @@ -507,6 +567,7 @@ String serializeResultRow(const ValidationResultRow& result) object->setProperty("path", result.path); object->setProperty("outcome", result.outcome); object->setProperty("reason", result.reason); + object->setProperty("retryable", result.retryable); return JSON::toString(var(object), false).replaceCharacters("\r\n", " "); } @@ -542,6 +603,7 @@ bool parseResultRowFromOutput(const String& output, ValidationResultRow& result) result.path = object->getProperty("path").toString(); result.outcome = object->getProperty("outcome").toString(); result.reason = object->getProperty("reason").toString(); + result.retryable = static_cast(object->getProperty("retryable")); return true; } @@ -550,42 +612,18 @@ bool parseResultRowFromOutput(const String& output, ValidationResultRow& result) bool isRetryableFailure(const ValidationResultRow& result) { - if (result.outcome != "failed") - return false; - - const auto& reason = result.reason; - return reason.containsIgnoreCase("sleeping") - || reason.containsIgnoreCase("restarting") - || reason.containsIgnoreCase("try again") - || reason.containsIgnoreCase("timed out") - || reason.containsIgnoreCase("503") - || reason.containsIgnoreCase("Unable to make POST request"); + return result.outcome == "failed" && result.retryable; } ValidationResultRow runEntryInChildProcessOnce(const ValidationEntry& entry) { + // The child inherits this process's environment (tokens, timeout, report + // dir); only the model id needs passing, and argv keeps secrets off the + // command line and works on Windows StringArray command; - command.add("env"); - command.add("HARP_MODEL_VALIDATION_CHILD=1"); - command.add("HARP_MODEL_VALIDATION_ID=" + entry.id); - command.add("HARP_MODEL_VALIDATION_TIMEOUT_MS=" + String(getPerModelTimeoutMs())); - - const auto reportDir = getReportDir(); - command.add("HARP_MODEL_VALIDATION_REPORT_DIR=" + reportDir.getFullPathName()); - - const auto hfToken = firstNonEmptyEnv({ "HARP_HUGGINGFACE_TOKEN", "HF_TOKEN" }); - if (hfToken.isNotEmpty()) - { - command.add("HARP_HUGGINGFACE_TOKEN=" + hfToken); - } - - const auto stabilityKey = firstNonEmptyEnv({ "HARP_STABILITY_API_KEY", "STABILITY_API_KEY" }); - if (stabilityKey.isNotEmpty()) - { - command.add("HARP_STABILITY_API_KEY=" + stabilityKey); - } - command.add(File::getSpecialLocation(File::currentExecutableFile).getFullPathName()); + command.add("--child"); + command.add(entry.id); ChildProcess child; @@ -594,13 +632,25 @@ ValidationResultRow runEntryInChildProcessOnce(const ValidationEntry& entry) return { entry.id, entry.name, entry.path, "failed", "failed to launch child process" }; } - if (! child.waitForProcessToFinish(getPerModelTimeoutMs())) + // Drain output while waiting — an undrained pipe blocks the child once the + // buffer fills, which would surface as a bogus timeout + String output; + std::thread outputReader([&child, &output] { output = child.readAllProcessOutput(); }); + + const bool finished = child.waitForProcessToFinish(getPerModelTimeoutMs()); + + if (! finished) { child.kill(); - return { entry.id, entry.name, entry.path, "failed", "model validation timed out" }; } - const auto output = child.readAllProcessOutput(); + outputReader.join(); + + if (! finished) + { + return { entry.id, entry.name, entry.path, "failed", "model validation timed out", true }; + } + ValidationResultRow result; if (parseResultRowFromOutput(output, result)) @@ -635,17 +685,39 @@ ValidationResultRow runEntryInChildProcess(const ValidationEntry& entry) return result; } + +int runChildMode(const String& childModelId) +{ + const auto entries = loadRemoteValidationEntries(childModelId); + + if (entries.size() != 1) + { + std::cerr << "Child mode: model id \"" << childModelId + << "\" did not match exactly one enabled remote registry entry.\n"; + return 1; + } + + const auto result = validateEntry(entries.front()); + std::cout << "RESULT_JSON: " << serializeResultRow(result) << '\n'; + return result.outcome == "failed" ? 1 : 0; +} } // namespace int main(int argc, char* argv[]) { - ignoreUnused(argc, argv); - ScopedJuceInitialiser_GUI scopedJuce; try { - const auto entries = loadRemoteValidationEntries(); + for (int i = 1; i + 1 < argc; ++i) + { + if (String(argv[i]) == "--child") + { + return runChildMode(String::fromUTF8(argv[i + 1])); + } + } + + const auto entries = loadRemoteValidationEntries(getSelectedModelId()); const auto reportDir = getReportDir(); if (entries.empty()) @@ -654,13 +726,6 @@ int main(int argc, char* argv[]) return 1; } - if (isChildMode()) - { - const auto result = validateEntry(entries.front()); - std::cout << "RESULT_JSON: " << serializeResultRow(result) << '\n'; - return result.outcome == "failed" ? 1 : 0; - } - const auto totalEntries = static_cast(entries.size()); std::cout << "\n"; @@ -685,57 +750,49 @@ int main(int argc, char* argv[]) const auto result = runEntryInChildProcess(entry); results.push_back(result); + // Write after every model so a killed run still leaves a report + writeReport(reportDir, results); + if (result.outcome == "passed") std::cout << "PASSED\n"; else if (result.outcome == "skipped") std::cout << "SKIPPED (" << result.reason << ")\n"; + else if (result.outcome == "inconclusive") + std::cout << "INCONCLUSIVE (" << result.reason << ")\n"; else std::cout << "FAILED\n"; } - int passed = 0; - int failed = 0; - int skipped = 0; - std::vector failedResults; - - for (const auto& result : results) - { - if (result.outcome == "passed") - passed += 1; - else if (result.outcome == "skipped") - skipped += 1; - else - { - failed += 1; - failedResults.push_back(&result); - } - } + const auto summary = summarize(results); std::cout << "\n"; std::cout << "========================================\n"; std::cout << " Summary\n"; std::cout << "========================================\n"; - std::cout << " Passed: " << passed << " / " << totalEntries << "\n"; - std::cout << " Failed: " << failed << " / " << totalEntries << "\n"; - std::cout << " Skipped: " << skipped << " / " << totalEntries << "\n"; + std::cout << " Passed: " << summary.passed << " / " << totalEntries << "\n"; + std::cout << " Failed: " << summary.failed << " / " << totalEntries << "\n"; + std::cout << " Skipped: " << summary.skipped << " / " << totalEntries << "\n"; + std::cout << " Inconclusive: " << summary.inconclusive << " / " << totalEntries << "\n"; std::cout << "----------------------------------------\n"; - if (! failedResults.empty()) + if (summary.failed > 0) { std::cout << "\n Failed models:\n\n"; - for (const auto* result : failedResults) + for (const auto& result : results) { - std::cout << " " << result->path << "\n"; - std::cout << " Reason: " << result->reason << "\n\n"; + if (result.outcome == "passed" || result.outcome == "skipped" + || result.outcome == "inconclusive") + continue; + + std::cout << " " << result.path << "\n"; + std::cout << " Reason: " << result.reason << "\n\n"; } } std::cout << "========================================\n\n"; - writeReport(reportDir, results); - - return failed > 0 ? 1 : 0; + return summary.failed > 0 ? 1 : 0; } catch (const std::exception& exception) {