Skip to content

Skip write-only flush on read-side catalog serialize #1030

Skip write-only flush on read-side catalog serialize

Skip write-only flush on read-side catalog serialize #1030

Workflow file for this run

name: Build-Test
on:
pull_request:
workflow_dispatch:
jobs:
wasm:
runs-on: ubuntu-latest
timeout-minutes: 20
env:
EMSDK_VERSION: 3.1.74
WABT_VERSION: 1.0.36
steps:
- uses: actions/checkout@v4
- name: Cache Emscripten SDK
uses: actions/cache@v4
with:
path: ${{ runner.tool_cache }}/emsdk
key: emsdk-${{ runner.os }}-${{ env.EMSDK_VERSION }}
- name: Install Emscripten SDK
run: |
EMSDK_DIR="${RUNNER_TOOL_CACHE}/emsdk"
if [ ! -d "$EMSDK_DIR/.git" ]; then
rm -rf "$EMSDK_DIR"
git clone https://github.com/emscripten-core/emsdk.git "$EMSDK_DIR"
fi
cd "$EMSDK_DIR"
git fetch --tags --quiet
./emsdk install "$EMSDK_VERSION"
./emsdk activate "$EMSDK_VERSION"
echo "EMSDK=$EMSDK_DIR" >> "$GITHUB_ENV"
- name: Cache WABT
uses: actions/cache@v4
with:
path: ${{ runner.tool_cache }}/wabt-${{ env.WABT_VERSION }}
key: wabt-${{ runner.os }}-${{ env.WABT_VERSION }}
- name: Install WABT
run: |
WABT_DIR="${RUNNER_TOOL_CACHE}/wabt-${WABT_VERSION}"
if [ ! -x "$WABT_DIR/bin/wasm-strip" ]; then
rm -rf "$WABT_DIR"
mkdir -p "$WABT_DIR"
curl -fsSL "https://github.com/WebAssembly/wabt/releases/download/${WABT_VERSION}/wabt-${WABT_VERSION}-ubuntu-20.04.tar.gz" \
| tar -xz --strip-components=1 -C "$WABT_DIR"
fi
echo "$WABT_DIR/bin" >> "$GITHUB_PATH"
- name: Configure
run: |
source "$EMSDK/emsdk_env.sh"
./configure
- name: Build generated SQLite files
run: |
make sqlite3.c sqlite3.h sqlite3ext.h
- name: Build native DoltLite
run: |
mkdir -p build
cd build
../configure
make doltlite
- name: Build DoltLite wasm targets
run: |
source "$EMSDK/emsdk_env.sh"
make -C ext/wasm fiddle npm
- name: Smoke test DoltLite wasm runtime
run: |
node ext/wasm/test-doltlite-smoke.mjs
- name: End-to-end test DoltLite wasm writes
run: |
node ext/wasm/test-doltlite-write-e2e.mjs
- name: Roundtrip exported wasm DB into native DoltLite
run: |
node ext/wasm/test-doltlite-export-roundtrip.mjs
- name: Smoke test the @dolthub/doltlite-wasm npm package
run: |
chmod +x packaging/npm/assemble.sh packaging/npm/test-package.sh
bash packaging/npm/test-package.sh
- name: Build wasm distribution zip
run: |
source "$EMSDK/emsdk_env.sh"
make -C ext/wasm dist
- name: Upload wasm artifacts
uses: actions/upload-artifact@v4
with:
name: wasm-build
path: |
ext/wasm/jswasm/*
ext/wasm/sqlite-wasm-*.zip
development-builds:
runs-on: ubuntu-latest
timeout-minutes: 20
env:
CFLAGS: -O2 -g -Werror
steps:
- uses: actions/checkout@v4
- name: Install dependencies
run: bash .github/scripts/install-apt-deps.sh tcl-dev build-essential zlib1g-dev
- name: Configure
run: |
mkdir -p build && cd build
TCL_CONFIG=$(find /usr -path '/usr/share/miniconda/*' -prune -o -name tclConfig.sh -print 2>/dev/null | head -1)
if [ -n "$TCL_CONFIG" ]; then
../configure --with-tcl=$(dirname $TCL_CONFIG)
else
../configure
fi
- name: Build development test configurations
run: |
cd build
make DOLTLITE_PROLLY=1 testfixture
# --buildonly: compile All-Debug / All-O0 only. Asserts are live in
# All-Debug, but no tests run here — see the smoke step below.
./testfixture ../test/testrunner.tcl --buildonly --jobs 2 mdevtest
# Release CI uses -DNDEBUG, so PROLLY_ASSERT_* and other assert() checks
# are compiled out. Catch write-txn / dirty-check contract bugs that only
# abort when asserts are live (e.g. read-side catalog serialize).
- name: Assert-enabled dirty-check smoke
env:
# Job-level CFLAGS is -O2; force a true debug compile line.
CFLAGS: -g -O0 -Werror
run: |
mkdir -p build-assert && cd build-assert
TCL_CONFIG=$(find /usr -path '/usr/share/miniconda/*' -prune -o -name tclConfig.sh -print 2>/dev/null | head -1)
if [ -n "$TCL_CONFIG" ]; then
../configure --with-debug --with-tcl=$(dirname $TCL_CONFIG)
else
../configure --with-debug
fi
if grep -E 'T\.cc \+= .* -DNDEBUG' Makefile; then
echo "::error::assert-enabled configure still sets -DNDEBUG"
exit 1
fi
if ! grep -E 'T\.cc \+= .* -DSQLITE_DEBUG=1' Makefile; then
echo "::error::assert-enabled configure did not set -DSQLITE_DEBUG=1"
exit 1
fi
make -j"$(nproc)" DOLTLITE_PROLLY=1 catalog_serialize_determinism_test
./catalog_serialize_determinism_test
build-and-test:
name: ${{ matrix.platform }}
runs-on: ${{ matrix.runs-on }}
timeout-minutes: 20
env:
CFLAGS: -O2 -g -Werror
strategy:
fail-fast: false
matrix:
include:
- platform: ubuntu
runs-on: ubuntu-latest
- platform: windows
runs-on: windows-latest
- platform: macos
runs-on: macos-latest
steps:
- uses: actions/checkout@v4
- name: Install dependencies
if: matrix.platform == 'ubuntu'
run: |
bash .github/scripts/install-apt-deps.sh tcl-dev build-essential zlib1g-dev ripgrep
- name: Install Windows dependencies
if: matrix.platform == 'windows'
uses: msys2/setup-msys2@v2
with:
msystem: MINGW64
install: mingw-w64-x86_64-gcc mingw-w64-x86_64-python mingw-w64-x86_64-tcl mingw-w64-x86_64-zlib make perl tcl openssl
- name: Install dependencies (macOS)
if: matrix.platform == 'macos'
run: brew install tcl-tk ripgrep
- name: Install Dolt
if: matrix.platform == 'ubuntu'
run: |
curl -fsSL https://github.com/dolthub/dolt/releases/latest/download/dolt-linux-amd64.tar.gz \
| sudo tar -xz -C /usr/local --strip-components=1
dolt version
- name: Configure
if: matrix.platform == 'ubuntu'
run: |
mkdir -p build && cd build
TCL_CONFIG=$(find /usr -path '/usr/share/miniconda/*' -prune -o -name tclConfig.sh -print 2>/dev/null | head -1)
if [ -n "$TCL_CONFIG" ]; then
../configure --with-tcl=$(dirname $TCL_CONFIG)
else
../configure
fi
- name: Configure Windows
if: matrix.platform == 'windows'
shell: msys2 {0}
run: |
mkdir -p build && cd build
../configure
- name: Configure (macOS)
if: matrix.platform == 'macos'
run: |
mkdir -p build && cd build
export PATH="$(brew --prefix tcl-tk)/bin:$PATH"
../configure
- name: Build
if: matrix.platform != 'windows'
run: |
cd build
set -o pipefail
{
make DOLTLITE_PROLLY_CHECK=1 doltlite
make DOLTLITE_PROLLY_CHECK=1 doltlite-remotesrv
make DOLTLITE_PROLLY_CHECK=1 doltlite-lib
# doltlite_parity.sh compares against the package-built stock
# SQLite CLI instead of whatever sqlite3 the runner image provides.
make DOLTLITE_PROLLY=0 sqlite3
# Stock SQLite lib for concurrent_branch_test
make libsqlite3.a USE_AMALGAMATION=0
} 2>&1 | tee build.log
DOLTLITE_CHECK_AMALGAMATION=0 bash ../test/assert_doltlite_artifacts.sh .
if grep -E "warning:" build.log; then
echo "::error::compiler warnings in the build (see above); the build must be warning-free"
exit 1
fi
- name: Build Windows
if: matrix.platform == 'windows'
shell: msys2 {0}
run: |
cd build
make -j$(nproc) DOLTLITE_PROLLY_CHECK=1 doltlite.exe doltlite-remotesrv.exe doltlite-lib
DOLTLITE_CHECK_AMALGAMATION=0 DOLTLITE_CHECK_SHARED=0 bash ../test/assert_doltlite_artifacts.sh .
cp -f "$(command -v sqlite3)" sqlite3
- name: Lint
if: matrix.platform != 'windows'
run: cd build && make lint
timeout-minutes: 1
- name: Basic smoke tests
if: matrix.platform != 'windows'
run: |
cd build
rm -f /tmp/build_test_smoke_basic.db /tmp/build_test_smoke_basic.db-wal /tmp/build_test_smoke_basic.db-lock
test "$(./doltlite :memory: "SELECT 1;")" = "1"
./doltlite /tmp/build_test_smoke_basic.db "CREATE TABLE t(id INTEGER PRIMARY KEY, v TEXT); INSERT INTO t VALUES(1,'a'); SELECT count(*) FROM t;"
test "$(./doltlite /tmp/build_test_smoke_basic.db "SELECT v FROM t WHERE id=1;")" = "a"
./doltlite /tmp/build_test_smoke_basic.db "SELECT dolt_commit('-A','-m','c1');"
test "$(./doltlite /tmp/build_test_smoke_basic.db "SELECT count(*) FROM dolt_log;")" = "2"
timeout-minutes: 2
- name: Basic smoke tests
if: matrix.platform == 'windows'
shell: msys2 {0}
run: |
cd build
rm -f /tmp/build_test_smoke_basic.db /tmp/build_test_smoke_basic.db-wal /tmp/build_test_smoke_basic.db-lock
test "$(./doltlite.exe :memory: "SELECT 1;")" = "1"
./doltlite.exe /tmp/build_test_smoke_basic.db "CREATE TABLE t(id INTEGER PRIMARY KEY, v TEXT); INSERT INTO t VALUES(1,'a'); SELECT count(*) FROM t;"
test "$(./doltlite.exe /tmp/build_test_smoke_basic.db "SELECT v FROM t WHERE id=1;")" = "a"
./doltlite.exe /tmp/build_test_smoke_basic.db "SELECT dolt_commit('-A','-m','c1');"
test "$(./doltlite.exe /tmp/build_test_smoke_basic.db "SELECT count(*) FROM dolt_log;")" = "2"
timeout-minutes: 2
- name: Windows path smoke tests
if: matrix.platform == 'windows'
shell: msys2 {0}
run: cd build && bash ../test/windows_path_smoke.sh ./doltlite.exe
timeout-minutes: 2
- name: GC smoke tests
if: matrix.platform != 'windows'
run: |
cd build
rm -f /tmp/build_test_smoke_gc.db /tmp/build_test_smoke_gc.db-wal /tmp/build_test_smoke_gc.db-lock
./doltlite /tmp/build_test_smoke_gc.db "CREATE TABLE t(id INTEGER PRIMARY KEY, v TEXT); INSERT INTO t VALUES(1,'a'); SELECT dolt_commit('-A','-m','c1'); SELECT dolt_gc(); SELECT count(*) FROM t;"
bash ../test/doltlite_gc.sh
bash ../test/doltlite_gc_session_state.sh
bash ../test/doltlite_pragma_page_count.sh
timeout-minutes: 5
- name: GC smoke tests
if: matrix.platform == 'windows'
shell: msys2 {0}
run: |
cd build
rm -f /tmp/build_test_smoke_gc.db /tmp/build_test_smoke_gc.db-wal /tmp/build_test_smoke_gc.db-lock
./doltlite.exe /tmp/build_test_smoke_gc.db "CREATE TABLE t(id INTEGER PRIMARY KEY, v TEXT); INSERT INTO t VALUES(1,'a'); SELECT dolt_commit('-A','-m','c1'); SELECT dolt_gc(); SELECT count(*) FROM t;"
bash ../test/doltlite_gc.sh
bash ../test/doltlite_gc_session_state.sh
bash ../test/doltlite_pragma_page_count.sh
timeout-minutes: 5
# -- Doltlite feature tests ------------------------------
- name: Doltlite feature tests
if: matrix.platform != 'windows'
run: cd build && bash ../test/run_doltlite_tests.sh
timeout-minutes: 20
- name: Correctness regression tests
if: matrix.platform != 'windows'
run: cd build && bash ../test/correctness_test.sh ./doltlite
timeout-minutes: 10
- name: Index operation tests
if: matrix.platform != 'windows'
run: cd build && bash ../test/index_test.sh ./doltlite
timeout-minutes: 5
# All oracle tests (vc + sql) run in the parallel oracle-tests job
- name: Large-scale test (10K rows)
if: matrix.platform != 'windows'
run: cd build && bash ../test/large_scale_test.sh ./doltlite --quick
timeout-minutes: 5
- name: Chunk size distribution
if: matrix.platform != 'windows'
run: cd build && bash ../test/chunk_distribution_test.sh ./doltlite
timeout-minutes: 5
- name: Chunker boundary golden
if: matrix.platform != 'windows'
run: cd build && bash ../test/chunker_boundary_golden.sh ./doltlite
timeout-minutes: 5
- name: Multi-table test
if: matrix.platform != 'windows'
run: cd build && bash ../test/multi_table_test.sh ./doltlite --quick
timeout-minutes: 5
- name: Diff tests
if: matrix.platform != 'windows'
run: cd build && bash ../test/diff_test.sh ./doltlite
timeout-minutes: 10
- name: Config tests
if: matrix.platform != 'windows'
run: cd build && bash ../test/config_test.sh ./doltlite
timeout-minutes: 2
- name: BLAKE3 known-answer test
if: matrix.platform != 'windows'
run: cd build && bash ../test/blake3_kat_test.sh ./doltlite
timeout-minutes: 2
- name: Credential known-answer test
if: matrix.platform != 'windows'
run: cd build && bash ../test/creds_kat_test.sh
timeout-minutes: 2
- name: TLS client test
if: matrix.platform != 'windows'
run: cd build && bash ../test/tls_test.sh
timeout-minutes: 2
- name: Windows Doltlite feature tests
if: matrix.platform == 'windows'
shell: msys2 {0}
run: |
cd build
export DLTEST_STRIP_CR=1
rm -rf /d/tmp
powershell.exe -NoProfile -Command "New-Item -ItemType Junction -Path 'D:\\tmp' -Target '$(cygpath -w /tmp)' | Out-Null"
cat > ./doltlite-no-msys-conv <<'SH'
#!/usr/bin/env bash
export MSYS2_ARG_CONV_EXCL='*'
exec ./doltlite "$@"
SH
chmod +x ./doltlite-no-msys-conv
export DOLTLITE=./doltlite-no-msys-conv
source ../test/lib/doltlite_suite_manifest.sh
while IFS= read -r test; do
test="../test/$test"
echo "━━━ ${test} ━━━"
bash "$test" "$DOLTLITE"
done < <(doltlite_windows_suites)
timeout-minutes: 20
- name: Windows correctness regression tests
if: matrix.platform == 'windows'
shell: msys2 {0}
run: |
cd build
bash ../test/correctness_test.sh ./doltlite
bash ../test/index_test.sh ./doltlite
timeout-minutes: 15
- name: Windows scale and feature tests
if: matrix.platform == 'windows'
shell: msys2 {0}
run: |
cd build
bash ../test/large_scale_test.sh ./doltlite --quick
bash ../test/chunk_distribution_test.sh ./doltlite
bash ../test/chunker_boundary_golden.sh ./doltlite
bash ../test/multi_table_test.sh ./doltlite --quick
bash ../test/diff_test.sh ./doltlite
bash ../test/config_test.sh ./doltlite
bash ../test/blake3_kat_test.sh ./doltlite
timeout-minutes: 20
- name: Windows credential + TLS + remote tests
if: matrix.platform == 'windows'
shell: msys2 {0}
run: |
cd build
bash ../test/creds_kat_test.sh
bash ../test/creds_path_test.sh "$DOLTLITE"
bash ../test/tls_test.sh
bash ../test/creds_verify_test.sh
bash ../test/remote_test.sh ./doltlite
bash ../test/remote_https_auth_test.sh ./doltlite.exe ./doltlite-remotesrv.exe
timeout-minutes: 15
- name: Windows HTTP remotesrv tests
if: matrix.platform == 'windows'
shell: msys2 {0}
run: |
cd build
bash ../test/http_remote_content_length_test.sh ./doltlite.exe ./doltlite-remotesrv.exe
bash ../test/remotesrv_http_test.sh ./doltlite.exe ./doltlite-remotesrv.exe
bash ../test/remotesrv_http_concurrency_test.sh ./doltlite.exe ./doltlite-remotesrv.exe
bash ../test/remotesrv_stall_test.sh ./doltlite.exe ./doltlite-remotesrv.exe
bash ../test/remotesrv_http_partial_failure_test.sh ./doltlite.exe ./doltlite-remotesrv.exe
bash ../test/remotesrv_http_force_push_test.sh ./doltlite.exe ./doltlite-remotesrv.exe
make DOLTLITE_PROLLY=1 DOLTLITE_PROLLY_CHECK=1 sqlite3.c sqlite3.h
bash ../test/amalgamation_windows_source_test.sh ./sqlite3.c
REMOTESRV=./doltlite-remotesrv.exe bash ../test/amalgamation_http_remote_test.sh .
timeout-minutes: 20
- name: Remote integration tests
if: matrix.platform != 'windows'
run: cd build && bash ../test/remote_test.sh ./doltlite
timeout-minutes: 5
- name: Credential verify known-answer test
if: matrix.platform != 'windows'
run: |
cd build
bash ../test/creds_verify_test.sh
bash ../test/creds_path_test.sh ./doltlite
timeout-minutes: 5
- name: Remote HTTPS + credential auth tests
if: matrix.platform != 'windows'
run: cd build && bash ../test/remote_https_auth_test.sh ./doltlite ./doltlite-remotesrv
timeout-minutes: 5
- name: HTTP remote Content-Length test
if: matrix.platform != 'windows'
run: cd build && bash ../test/http_remote_content_length_test.sh ./doltlite ./doltlite-remotesrv
timeout-minutes: 5
- name: HTTP remote lifecycle test (strict proxy)
if: matrix.platform != 'windows'
run: cd build && bash ../test/remotesrv_http_test.sh ./doltlite ./doltlite-remotesrv
timeout-minutes: 5
- name: HTTP remote concurrency tests
if: matrix.platform != 'windows'
run: cd build && bash ../test/remotesrv_http_concurrency_test.sh ./doltlite ./doltlite-remotesrv
timeout-minutes: 5
- name: HTTP remote stalled client tests
if: matrix.platform != 'windows'
run: cd build && bash ../test/remotesrv_stall_test.sh ./doltlite ./doltlite-remotesrv
timeout-minutes: 5
- name: HTTP remote partial failure tests
if: matrix.platform != 'windows'
run: cd build && bash ../test/remotesrv_http_partial_failure_test.sh ./doltlite ./doltlite-remotesrv
timeout-minutes: 5
- name: HTTP remote force push tests
if: matrix.platform != 'windows'
run: cd build && bash ../test/remotesrv_http_force_push_test.sh ./doltlite ./doltlite-remotesrv
timeout-minutes: 5
- name: Amalgamation plaintext HTTP remote test
if: matrix.platform != 'windows'
run: |
cd build
rm -f sqlite3.c .target_source
make DOLTLITE_PROLLY=1 DOLTLITE_PROLLY_CHECK=1 sqlite3.c sqlite3.h
bash ../test/amalgamation_http_remote_test.sh .
timeout-minutes: 5
# -- C/Python/Go integration tests -----------------------
- name: C integration tests
if: matrix.platform != 'windows'
run: |
cd build
# Quickstart
gcc $CFLAGS -o quickstart ../examples/quickstart.c -I. libdoltlite.a -lpthread -lz -lm
./quickstart
# Concurrent branch test
cc $CFLAGS -g -O0 -I. -I../src -DDOLTLITE_PROLLY=1 -D_HAVE_SQLITE_CONFIG_H -DBUILD_sqlite \
-o concurrent_branch_test ../test/concurrent_branch_test.c \
libsqlite3.a -lz -lpthread -lm
./concurrent_branch_test
- name: Concurrent write test
if: matrix.platform != 'windows'
run: |
cd build
gcc $CFLAGS -g -O0 -I. -o concurrent_write_test ../test/concurrent_write_test.c \
libdoltlite.a -lz -lpthread -lm
./concurrent_write_test
- name: Native crash recovery test
if: matrix.platform == 'ubuntu'
run: |
cd build
cc $CFLAGS -I. -o crash_recovery_test ../test/crash_recovery_test.c \
libdoltlite.a -lz -lpthread -lm
./crash_recovery_test
timeout-minutes: 10
- name: Prepared statement reuse test
if: matrix.platform != 'windows'
run: |
cd build
gcc $CFLAGS -g -O0 -I. -o prepared_stmt_reuse_test ../test/prepared_stmt_reuse_test.c \
libdoltlite.a -lz -lpthread -lm
./prepared_stmt_reuse_test
- name: Windows native C regression tests
if: matrix.platform == 'windows'
shell: msys2 {0}
run: cd build && bash ../test/run_doltlite_regression_case.sh all_except=backup_safety
timeout-minutes: 15
# Wires the previously-orphan C tests into CI. All tests in
# test/run_c_tests.sh are gating and must pass.
- name: Warning-free orphan C tests
if: matrix.platform == 'ubuntu'
run: cd build && make c-tests
timeout-minutes: 15
- name: Python integration test
if: matrix.platform == 'ubuntu'
run: cd build && LD_PRELOAD=./libdoltlite.so python3 ../examples/quickstart.py
- name: Go integration test
if: matrix.platform == 'ubuntu'
uses: actions/setup-go@v5
with:
go-version: '1.23'
- if: matrix.platform == 'ubuntu'
run: |
cd examples/go
CGO_CFLAGS="-I../../build" CGO_LDFLAGS="../../build/libdoltlite.a -lz -lpthread -lm" \
go build -tags libsqlite3 -o quickstart .
./quickstart