diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a9aa7507..f36a8917 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -85,7 +85,9 @@ jobs: uses: actions/upload-artifact@v4 with: name: an4-logdir-${{ github.job }} - path: an4/logdir + path: | + an4/logdir + an4/qmanager train-installed: runs-on: ubuntu-latest @@ -118,7 +120,9 @@ jobs: uses: actions/upload-artifact@v4 with: name: an4-logdir-${{ github.job }} - path: an4/logdir + path: | + an4/logdir + an4/qmanager train-parallel: runs-on: ubuntu-latest @@ -161,7 +165,9 @@ jobs: uses: actions/upload-artifact@v4 with: name: an4-logdir-${{ github.job }} - path: an4/logdir + path: | + an4/logdir + an4/qmanager train-align: runs-on: ubuntu-latest @@ -195,7 +201,9 @@ jobs: uses: actions/upload-artifact@v4 with: name: an4-logdir-${{ github.job }} - path: an4/logdir + path: | + an4/logdir + an4/qmanager train-g2p-lda-vtln: runs-on: ubuntu-22.04 # Requires libngram-dev for G2P support @@ -244,7 +252,9 @@ jobs: uses: actions/upload-artifact@v4 with: name: an4-logdir-${{ github.job }} - path: an4/logdir + path: | + an4/logdir + an4/qmanager train-semi: runs-on: ubuntu-latest @@ -281,7 +291,9 @@ jobs: uses: actions/upload-artifact@v4 with: name: an4-logdir-${{ github.job }} - path: an4/logdir + path: | + an4/logdir + an4/qmanager train-ptm: runs-on: ubuntu-latest @@ -318,4 +330,6 @@ jobs: uses: actions/upload-artifact@v4 with: name: an4-logdir-${{ github.job }} - path: an4/logdir + path: | + an4/logdir + an4/qmanager diff --git a/scripts/000.comp_feat/make_feats.pl b/scripts/000.comp_feat/make_feats.pl index 20635b48..4b65c2cd 100644 --- a/scripts/000.comp_feat/make_feats.pl +++ b/scripts/000.comp_feat/make_feats.pl @@ -95,7 +95,16 @@ chomp; push @ctl_lines, $_; my $dir = dirname($_); - mkpath(catdir($ST::CFG_FEATFILES_DIR, $dir)); + # Create the output subdirectory for this utterance under $outfolder, so a + # warp pass lands in feat// rather than the base feature directory. + # Sibling parts run in parallel and may create the same directory at the + # same time; mkpath() is not atomic, so tolerate an "already exists" race + # instead of dying (which previously killed the whole comp_feat stage). + my $target = catdir($outfolder, $dir); + unless (-d $target) { + eval { mkpath($target) }; + die "Failed to create $target: $@" if $@ and not -d $target; + } } close CTL; diff --git a/scripts/lib/SphinxTrain/Resolved.pm b/scripts/lib/SphinxTrain/Resolved.pm index 50d89f4f..ddb131a9 100644 --- a/scripts/lib/SphinxTrain/Resolved.pm +++ b/scripts/lib/SphinxTrain/Resolved.pm @@ -204,9 +204,20 @@ sub read_document { sub write_file { my ($path, $cfg_path) = @_; my $doc = build_document($cfg_path); - open my $fh, ">", $path or die "Cannot write $path: $!\n"; + # Write atomically. sync_runtime() regenerates this file from every script + # that loads the config, so parallel shards (e.g. 000.comp_feat) can enter + # write_file() at the same time. A plain open(">", $path) truncates in + # place *and* bumps the file's mtime, so a sibling that then judges the file + # fresh reads it while it is still empty and dies "malformed JSON ... (end + # of string)", failing the whole stage. Write a per-process temp file in + # the same directory and rename() it over the target (atomic on POSIX): a + # reader always sees either the old or the new complete file, never a + # truncated one. + my $tmp = "$path.tmp.$$"; + open my $fh, ">", $tmp or die "Cannot write $tmp: $!\n"; print {$fh} to_json($doc); - close $fh or die "Cannot close $path: $!\n"; + close $fh or die "Cannot close $tmp: $!\n"; + rename($tmp, $path) or die "Cannot rename $tmp to $path: $!\n"; return $doc; } diff --git a/scripts/lib/SphinxTrain/Util.pm b/scripts/lib/SphinxTrain/Util.pm index 1096903f..e9a482ef 100644 --- a/scripts/lib/SphinxTrain/Util.pm +++ b/scripts/lib/SphinxTrain/Util.pm @@ -463,8 +463,22 @@ sub WaitForScript { $exit = $id >> 8; } else { - $ST::Q->waitfor_job($id); - $exit = $? >> 8; + my $reaped = $ST::Q->waitfor_job($id); + # $? only describes our child when waitpid() actually reaped it. + # waitpid() returns -1 when the child was already reaped (or is not + # ours); there is no status to read, so do not manufacture a + # failure out of a stale $? -- that is how an already-finished job + # became a spurious "exit code 255" (-1 >> 8 == -1). A signal + # death is reported as 128+signo instead of being masked to 0. + if (defined($reaped) && $reaped == -1) { + $exit = 0; + } + elsif ($? & 127) { + $exit = 128 + ($? & 127); + } + else { + $exit = $? >> 8; + } } if ($exit != 0) { LogError("Parallel job failed with exit code $exit\n");