comp_feat: fix flaky parallel exit-255 (atomic resolved-config write)

.github/workflows/tests.yml

@@ -85,7 +85,9 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: an4-logdir-${{ github.job }}
-          path: an4/logdir
+          path: |
+            an4/logdir
+            an4/qmanager
 
   train-installed:
     runs-on: ubuntu-latest
@@ -118,7 +120,9 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: an4-logdir-${{ github.job }}
-          path: an4/logdir
+          path: |
+            an4/logdir
+            an4/qmanager
 
   train-parallel:
     runs-on: ubuntu-latest
@@ -161,7 +165,9 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: an4-logdir-${{ github.job }}
-          path: an4/logdir
+          path: |
+            an4/logdir
+            an4/qmanager
 
   train-align:
     runs-on: ubuntu-latest
@@ -195,7 +201,9 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: an4-logdir-${{ github.job }}
-          path: an4/logdir
+          path: |
+            an4/logdir
+            an4/qmanager
 
   train-g2p-lda-vtln:
     runs-on: ubuntu-22.04  # Requires libngram-dev for G2P support
@@ -244,7 +252,9 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: an4-logdir-${{ github.job }}
-          path: an4/logdir
+          path: |
+            an4/logdir
+            an4/qmanager
 
   train-semi:
     runs-on: ubuntu-latest
@@ -281,7 +291,9 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: an4-logdir-${{ github.job }}
-          path: an4/logdir
+          path: |
+            an4/logdir
+            an4/qmanager
 
   train-ptm:
     runs-on: ubuntu-latest
@@ -318,4 +330,6 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: an4-logdir-${{ github.job }}
-          path: an4/logdir
+          path: |
+            an4/logdir
+            an4/qmanager

scripts/000.comp_feat/make_feats.pl

@@ -95,7 +95,16 @@
     chomp;
     push @ctl_lines, $_;
     my $dir = dirname($_);
-    mkpath(catdir($ST::CFG_FEATFILES_DIR, $dir));
+    # Create the output subdirectory for this utterance under $outfolder, so a
+    # warp pass lands in feat/<warp>/ rather than the base feature directory.
+    # Sibling parts run in parallel and may create the same directory at the
+    # same time; mkpath() is not atomic, so tolerate an "already exists" race
+    # instead of dying (which previously killed the whole comp_feat stage).
+    my $target = catdir($outfolder, $dir);
+    unless (-d $target) {
+        eval { mkpath($target) };
+        die "Failed to create $target: $@" if $@ and not -d $target;
+    }
 }
 close CTL;
 

scripts/lib/SphinxTrain/Resolved.pm

@@ -204,9 +204,20 @@ sub read_document {
 sub write_file {
     my ($path, $cfg_path) = @_;
     my $doc = build_document($cfg_path);
-    open my $fh, ">", $path or die "Cannot write $path: $!\n";
+    # Write atomically.  sync_runtime() regenerates this file from every script
+    # that loads the config, so parallel shards (e.g. 000.comp_feat) can enter
+    # write_file() at the same time.  A plain open(">", $path) truncates in
+    # place *and* bumps the file's mtime, so a sibling that then judges the file
+    # fresh reads it while it is still empty and dies "malformed JSON ... (end
+    # of string)", failing the whole stage.  Write a per-process temp file in
+    # the same directory and rename() it over the target (atomic on POSIX): a
+    # reader always sees either the old or the new complete file, never a
+    # truncated one.
+    my $tmp = "$path.tmp.$$";
+    open my $fh, ">", $tmp or die "Cannot write $tmp: $!\n";
     print {$fh} to_json($doc);
-    close $fh or die "Cannot close $path: $!\n";
+    close $fh or die "Cannot close $tmp: $!\n";
+    rename($tmp, $path) or die "Cannot rename $tmp to $path: $!\n";
     return $doc;
 }
 

scripts/lib/SphinxTrain/Util.pm

@@ -463,8 +463,22 @@ sub WaitForScript {
             $exit = $id >> 8;
         }
         else {
-            $ST::Q->waitfor_job($id);
-            $exit = $? >> 8;
+            my $reaped = $ST::Q->waitfor_job($id);
+            # $? only describes our child when waitpid() actually reaped it.
+            # waitpid() returns -1 when the child was already reaped (or is not
+            # ours); there is no status to read, so do not manufacture a
+            # failure out of a stale $? -- that is how an already-finished job
+            # became a spurious "exit code 255" (-1 >> 8 == -1).  A signal
+            # death is reported as 128+signo instead of being masked to 0.
+            if (defined($reaped) && $reaped == -1) {
+                $exit = 0;
+            }
+            elsif ($? & 127) {
+                $exit = 128 + ($? & 127);
+            }
+            else {
+                $exit = $? >> 8;
+            }
         }
         if ($exit != 0) {
             LogError("Parallel job failed with exit code $exit\n");