Optimize archive import and runtime logging

.github/workflows/ci.yml

@@ -11,13 +11,27 @@ on:
 jobs:
   smoke-windows:
     runs-on: windows-latest
+    timeout-minutes: 60
+    env:
+      VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}\\.vcpkg-cache
+      VCPKG_BINARY_SOURCES: clear;files,${{ github.workspace }}\\.vcpkg-cache,readwrite
 
     steps:
       - name: Checkout
         uses: actions/checkout@v4
         with:
           submodules: recursive
 
+      - name: Cache vcpkg artifacts
+        uses: actions/cache@v4
+        with:
+          path: |
+            vcpkg/downloads
+            .vcpkg-cache
+          key: vcpkg-${{ runner.os }}-x64-windows-${{ hashFiles('vcpkg.json') }}
+          restore-keys: |
+            vcpkg-${{ runner.os }}-x64-windows-
+
       - name: Setup CMake
         uses: jwlawson/actions-setup-cmake@v2
 
@@ -40,17 +54,31 @@ jobs:
 
       - name: Test (smoke label)
         shell: pwsh
-        run: ctest --preset test-ci-debug -L smoke
+        run: ctest --preset test-ci-debug -L smoke --output-on-failure
 
   edge-windows:
     runs-on: windows-latest
+    timeout-minutes: 90
+    env:
+      VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}\\.vcpkg-cache
+      VCPKG_BINARY_SOURCES: clear;files,${{ github.workspace }}\\.vcpkg-cache,readwrite
 
     steps:
       - name: Checkout
         uses: actions/checkout@v4
         with:
           submodules: recursive
 
+      - name: Cache vcpkg artifacts
+        uses: actions/cache@v4
+        with:
+          path: |
+            vcpkg/downloads
+            .vcpkg-cache
+          key: vcpkg-${{ runner.os }}-x64-windows-${{ hashFiles('vcpkg.json') }}
+          restore-keys: |
+            vcpkg-${{ runner.os }}-x64-windows-
+
       - name: Setup CMake
         uses: jwlawson/actions-setup-cmake@v2
 
@@ -81,4 +109,4 @@ jobs:
 
       - name: Test (edge label)
         shell: pwsh
-        run: ctest --preset test-ci-debug -L edge
+        run: ctest --preset test-ci-debug -L edge --output-on-failure

.github/workflows/cmake-multi-platform.yml

@@ -6,6 +6,10 @@ on:
 jobs:
   build-test:
     runs-on: ${{ matrix.os }}
+    timeout-minutes: 120
+    env:
+      VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/.vcpkg-cache
+      VCPKG_BINARY_SOURCES: clear;files,${{ github.workspace }}/.vcpkg-cache,readwrite
 
     strategy:
       fail-fast: false
@@ -27,6 +31,16 @@ jobs:
       - name: Setup CMake
         uses: jwlawson/actions-setup-cmake@v2
 
+      - name: Cache vcpkg artifacts
+        uses: actions/cache@v4
+        with:
+          path: |
+            vcpkg/downloads
+            .vcpkg-cache
+          key: vcpkg-${{ runner.os }}-${{ matrix.triplet }}-${{ hashFiles('vcpkg.json') }}
+          restore-keys: |
+            vcpkg-${{ runner.os }}-${{ matrix.triplet }}-
+
       - name: Setup MSVC environment
         if: runner.os == 'Windows'
         uses: ilammy/msvc-dev-cmd@v1
@@ -58,4 +72,4 @@ jobs:
 
       - name: Test
         shell: bash
-        run: ctest --test-dir build-matrix --output-on-failure
+        run: ctest --test-dir build-matrix -L smoke --output-on-failure

CMakeLists.txt

@@ -1,6 +1,36 @@
 cmake_minimum_required(VERSION 3.20)
 
-project(DynamicFileBinder VERSION 0.1.0 LANGUAGES CXX)
+file(READ "${CMAKE_SOURCE_DIR}/VERSION" DFB_VERSION_BASE_RAW)
+string(STRIP "${DFB_VERSION_BASE_RAW}" DFB_VERSION_BASE)
+if (DFB_VERSION_BASE STREQUAL "")
+    set(DFB_VERSION_BASE "0.0.0")
+endif()
+
+string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" DFB_VERSION_PROJECT "${DFB_VERSION_BASE}")
+if (DFB_VERSION_PROJECT STREQUAL "")
+    set(DFB_VERSION_PROJECT "0.0.0")
+endif()
+
+project(DynamicFileBinder VERSION ${DFB_VERSION_PROJECT} LANGUAGES CXX)
+
+string(RANDOM LENGTH 8 ALPHABET 0123456789ABCDEF DFB_BUILD_UUID_PART1)
+string(RANDOM LENGTH 4 ALPHABET 0123456789ABCDEF DFB_BUILD_UUID_PART2)
+string(RANDOM LENGTH 4 ALPHABET 0123456789ABCDEF DFB_BUILD_UUID_PART3)
+string(RANDOM LENGTH 4 ALPHABET 0123456789ABCDEF DFB_BUILD_UUID_PART4)
+string(RANDOM LENGTH 12 ALPHABET 0123456789ABCDEF DFB_BUILD_UUID_PART5)
+set(DFB_BUILD_ID "{${DFB_BUILD_UUID_PART1}-${DFB_BUILD_UUID_PART2}-${DFB_BUILD_UUID_PART3}-${DFB_BUILD_UUID_PART4}-${DFB_BUILD_UUID_PART5}}")
+set(DFB_BUILD_VERSION "${DFB_VERSION_BASE}+${DFB_BUILD_ID}")
+
+file(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/generated")
+file(WRITE "${CMAKE_BINARY_DIR}/generated/version_info.hpp"
+"#pragma once\n"
+"namespace dynamic_file_binder {\n"
+"inline constexpr const char* kVersionBase = \"${DFB_VERSION_BASE}\";\n"
+"inline constexpr const char* kBuildId = \"${DFB_BUILD_ID}\";\n"
+"inline constexpr const char* kVersion = \"${DFB_BUILD_VERSION}\";\n"
+"}\n")
+
+message(STATUS "DynamicFileBinder version: ${DFB_BUILD_VERSION}")
 
 include(CTest)
 find_package(wxWidgets REQUIRED COMPONENTS core base ribbon aui)
@@ -23,6 +53,7 @@ add_executable(dynamic_file_binder
     src/file_binder.cpp
     src/error.cpp
     src/governance.cpp
+    src/job_queue_controller.cpp
     src/ui.cpp
     src/ui_wx.cpp
     src/database.cpp
@@ -32,11 +63,17 @@ add_executable(dynamic_file_binder
     src/context_menu.cpp
     src/daemon.cpp
     src/link_service.cpp
+    src/job_archive_format.cpp
 )
 
+if (WIN32)
+    set_target_properties(dynamic_file_binder PROPERTIES WIN32_EXECUTABLE TRUE)
+endif()
+
 target_include_directories(dynamic_file_binder
     PRIVATE
         include
+        ${CMAKE_BINARY_DIR}/generated
 )
 dfb_configure_output_dirs(dynamic_file_binder)
 
@@ -69,11 +106,15 @@ add_executable(dynamic_file_binder_tests
     src/settings.cpp
     src/system_monitor.cpp
     src/link_service.cpp
+    src/daemon.cpp
+    src/job_queue_controller.cpp
+    src/job_archive_format.cpp
 )
 
 target_include_directories(dynamic_file_binder_tests
     PRIVATE
         include
+        ${CMAKE_BINARY_DIR}/generated
 )
 dfb_configure_output_dirs(dynamic_file_binder_tests)
 
@@ -90,6 +131,26 @@ if (WIN32)
 endif()
 
 add_test(NAME core.error COMMAND dynamic_file_binder_tests error)
+add_test(NAME core.focus_logger COMMAND dynamic_file_binder_tests focus_logger)
+add_test(NAME core.focus_error_log_bridge COMMAND dynamic_file_binder_tests focus_error_log_bridge)
+add_test(NAME core.focus_hash_and_client_rights_logging COMMAND dynamic_file_binder_tests focus_hash_and_client_rights_logging)
+add_test(NAME core.focus_subsystem_success_events COMMAND dynamic_file_binder_tests focus_subsystem_success_events)
+add_test(NAME core.focus_cli_workflow_no_ui COMMAND dynamic_file_binder_tests focus_cli_workflow_no_ui)
+add_test(NAME core.focus_db_corrupt_rows COMMAND dynamic_file_binder_tests focus_db_corrupt_rows)
+add_test(NAME core.focus_ui_tray_startup_cycle COMMAND dynamic_file_binder_tests focus_ui_tray_startup_cycle)
+add_test(NAME core.focus_ui_tray_cli_launch COMMAND dynamic_file_binder_tests focus_ui_tray_cli_launch)
+add_test(NAME core.focus_ui_save_missing_source COMMAND dynamic_file_binder_tests focus_ui_save_missing_source)
+add_test(NAME core.focus_ui_save_valid_source COMMAND dynamic_file_binder_tests focus_ui_save_valid_source)
+add_test(NAME core.focus_queue_dedupe_duplicate_jobs COMMAND dynamic_file_binder_tests focus_queue_dedupe_duplicate_jobs)
+add_test(NAME core.focus_job_archive_roundtrip COMMAND dynamic_file_binder_tests focus_job_archive_roundtrip)
+add_test(NAME core.focus_job_archive_import_matrix COMMAND dynamic_file_binder_tests focus_job_archive_import_matrix)
+add_test(NAME core.focus_job_archive_json_matrix COMMAND dynamic_file_binder_tests focus_job_archive_json_matrix)
+add_test(NAME core.focus_ui_queue_start_no_crash COMMAND dynamic_file_binder_tests focus_ui_queue_start_no_crash)
+add_test(NAME core.focus_daemon_status_recovery COMMAND dynamic_file_binder_tests focus_daemon_status_recovery)
+add_test(NAME core.focus_daemon_stop_stale_pid COMMAND dynamic_file_binder_tests focus_daemon_stop_stale_pid)
+add_test(NAME core.focus_cleanup_missing_source COMMAND dynamic_file_binder_tests focus_cleanup_missing_source)
+add_test(NAME core.focus_cleanup_source_mismatch COMMAND dynamic_file_binder_tests focus_cleanup_source_mismatch)
+add_test(NAME core.focus_governance_revert_mismatch COMMAND dynamic_file_binder_tests focus_governance_revert_mismatch)
 add_test(NAME core.governance COMMAND dynamic_file_binder_tests governance)
 add_test(NAME core.governance_reject_physical COMMAND dynamic_file_binder_tests governance_reject_physical)
 add_test(NAME core.database COMMAND dynamic_file_binder_tests database)
@@ -101,10 +162,18 @@ add_test(NAME core.bind_preserve COMMAND dynamic_file_binder_tests bind_preserve
 add_test(NAME core.db_version COMMAND dynamic_file_binder_tests db_version)
 add_test(NAME core.revert_mismatch COMMAND dynamic_file_binder_tests revert_mismatch)
 add_test(NAME core.sync_lifecycle COMMAND dynamic_file_binder_tests sync_lifecycle)
+add_test(NAME core.sync_conflict COMMAND dynamic_file_binder_tests sync_conflict)
+add_test(NAME core.sync_stale_db COMMAND dynamic_file_binder_tests sync_stale_db)
 add_test(NAME core.state_storm COMMAND dynamic_file_binder_tests state_storm)
 add_test(NAME core.longrun COMMAND dynamic_file_binder_tests longrun)
 set_tests_properties(
     core.error
+    core.focus_logger
+    core.focus_error_log_bridge
+    core.focus_hash_and_client_rights_logging
+    core.focus_subsystem_success_events
+    core.focus_cli_workflow_no_ui
+    core.focus_db_corrupt_rows
     core.governance
     core.governance_reject_physical
     core.database
@@ -117,8 +186,49 @@ set_tests_properties(
     core.revert_mismatch
     PROPERTIES LABELS "smoke"
 )
-set_tests_properties(core.sync_lifecycle core.state_storm PROPERTIES LABELS "edge")
+set_tests_properties(
+    core.focus_logger
+    core.focus_error_log_bridge
+    core.focus_hash_and_client_rights_logging
+    core.focus_subsystem_success_events
+    core.focus_cli_workflow_no_ui
+    core.focus_db_corrupt_rows
+    core.focus_ui_tray_startup_cycle
+    core.focus_ui_tray_cli_launch
+    core.focus_ui_save_missing_source
+    core.focus_ui_save_valid_source
+    core.focus_queue_dedupe_duplicate_jobs
+    core.focus_job_archive_roundtrip
+    core.focus_job_archive_import_matrix
+    core.focus_job_archive_json_matrix
+    core.focus_ui_queue_start_no_crash
+    core.focus_daemon_status_recovery
+    core.focus_daemon_stop_stale_pid
+    core.focus_cleanup_missing_source
+    core.focus_cleanup_source_mismatch
+    core.focus_governance_revert_mismatch
+    PROPERTIES LABELS "focus"
+)
+set_tests_properties(core.focus_hash_and_client_rights_logging core.focus_subsystem_success_events core.focus_cli_workflow_no_ui core.focus_ui_tray_startup_cycle core.focus_ui_tray_cli_launch core.focus_ui_save_missing_source core.focus_ui_save_valid_source core.focus_queue_dedupe_duplicate_jobs core.focus_ui_queue_start_no_crash core.focus_daemon_status_recovery core.focus_daemon_stop_stale_pid core.focus_cleanup_missing_source core.focus_cleanup_source_mismatch core.focus_governance_revert_mismatch PROPERTIES TIMEOUT 30)
+set_tests_properties(core.sync_lifecycle core.sync_conflict core.sync_stale_db core.state_storm PROPERTIES LABELS "edge")
 set_tests_properties(core.longrun PROPERTIES TIMEOUT 180 LABELS "soak;edge")
+set_tests_properties(
+    core.focus_cli_workflow_no_ui
+    core.focus_ui_tray_startup_cycle
+    core.focus_ui_tray_cli_launch
+    core.focus_ui_save_missing_source
+    core.focus_ui_save_valid_source
+    core.focus_ui_queue_start_no_crash
+    core.focus_job_archive_roundtrip
+    core.focus_job_archive_import_matrix
+    core.focus_job_archive_json_matrix
+    core.focus_daemon_status_recovery
+    core.sync_lifecycle
+    core.sync_conflict
+    core.sync_stale_db
+    core.state_storm
+    PROPERTIES LABELS "full-e2e"
+)
 
 add_test(
     NAME soak.longrun_10min

CMakePresets.json

@@ -49,13 +49,39 @@
         "outputOnFailure": true
       }
     },
+    {
+      "name": "test-full-e2e-debug",
+      "configurePreset": "vs-debug-vcpkg",
+      "configuration": "Debug",
+      "filter": {
+        "label": {
+          "include": "full-e2e"
+        }
+      },
+      "output": {
+        "outputOnFailure": true
+      }
+    },
     {
       "name": "test-ci-debug",
       "configurePreset": "ci-windows",
       "configuration": "Debug",
       "output": {
         "outputOnFailure": true
       }
+    },
+    {
+      "name": "test-ci-full-e2e-debug",
+      "configurePreset": "ci-windows",
+      "configuration": "Debug",
+      "filter": {
+        "label": {
+          "include": "full-e2e"
+        }
+      },
+      "output": {
+        "outputOnFailure": true
+      }
     }
   ]
 }

VERSION

@@ -0,0 +1 @@
+0.1.0-alpha