Add Ruby version 4.0 to the test matrix

.github/workflows/test-suite.yaml

@@ -47,6 +47,14 @@ jobs:
             label: '3.3.0 (first release)'
           - ruby-version: '3.3'
             label: '3.3 (latest patch)'
+          - ruby-version: '3.4.1'
+            label: '3.4.1 (second release)'
+          - ruby-version: '3.4'
+            label: '3.4 (latest patch)'
+          - ruby-version: '4.0.0'
+            label: '4.0.0 (first release)'
+          - ruby-version: '4.0'
+            label: '4.0 (latest patch)'
     steps:
       -
         name: Checkout repository
@@ -91,10 +99,10 @@ jobs:
       -
         name: Install Ruby ${{ matrix.ruby-version }} and dependencies
         run: |
-          docker exec frontend bash -c "sudo apt-get update && sudo apt-get install -y build-essential libssl-dev libreadline-dev zlib1g-dev libxml2-dev curl git"
-          docker exec frontend bash -c "git clone https://github.com/rbenv/ruby-build.git /tmp/ruby-build"
-          docker exec frontend bash -c "sudo /tmp/ruby-build/install.sh"
-          docker exec frontend bash -c '
+          docker exec rocoto-frontend bash -c "sudo apt-get update && sudo apt-get install -y build-essential libssl-dev libreadline-dev zlib1g-dev libxml2-dev curl git"
+          docker exec rocoto-frontend bash -c "git clone https://github.com/rbenv/ruby-build.git /tmp/ruby-build"
+          docker exec rocoto-frontend bash -c "sudo /tmp/ruby-build/install.sh"
+          docker exec rocoto-frontend bash -c '
             VERSION="${{ matrix.ruby-version }}";
             if [[ $VERSION =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
               # Full version specified (e.g., 3.2.0)
@@ -110,27 +118,27 @@ jobs:
             sudo ln -sf $INSTALL_DIR/bin/ruby /usr/local/bin/ruby;
             sudo ln -sf $INSTALL_DIR/bin/gem /usr/local/bin/gem;
           '
-          docker exec frontend bash -c "ruby --version"
+          docker exec rocoto-frontend bash -c "ruby --version"
       -
         name: Copy test files to work directory
         run: |
-          docker exec frontend bash -l -c "mkdir -p work; cd work; cp -r ../rocoto/* ."
+          docker exec rocoto-frontend bash -l -c "mkdir -p work; cd work; cp -r ../rocoto/* ."
       -
         name: Install rocoto
         run: |
           # Extract minor version (X.Y) from matrix version for directory path
           MINOR_VERSION=$(echo "${{ matrix.ruby-version }}" | cut -d. -f1,2)
-          docker exec frontend bash -l -c "cd work; ./INSTALL --with-ruby=/usr/local/ruby-${MINOR_VERSION} --local"
-          docker exec frontend bash -l -c "cd work; ./UNINSTALL"
-          docker exec frontend bash -l -c "cd work; ./INSTALL --with-ruby=/usr/local/ruby-${MINOR_VERSION}"
+          docker exec rocoto-frontend bash -l -c "cd work; ./INSTALL --with-ruby=/usr/local/ruby-${MINOR_VERSION} --local"
+          docker exec rocoto-frontend bash -l -c "cd work; ./UNINSTALL"
+          docker exec rocoto-frontend bash -l -c "cd work; ./INSTALL --with-ruby=/usr/local/ruby-${MINOR_VERSION}"
       -
         name: Run test suite
         run: |
-          docker exec frontend bash -l -c "cd work; bin/rocotorun -h"
+          docker exec rocoto-frontend bash -l -c "cd work; bin/rocotorun -h"
       -
         name: Run RSpec tests
         run: |
-          docker exec frontend bash -l -c "BUNDLE_CMD=\$(ruby -e \"require 'rubygems'; puts Gem.bin_path('bundler', 'bundle')\"); cd work; \$BUNDLE_CMD exec rake spec"
+          docker exec rocoto-frontend bash -l -c "BUNDLE_CMD=\$(ruby -e \"require 'rubygems'; puts Gem.bin_path('bundler', 'bundle')\"); cd work; \$BUNDLE_CMD exec rake spec"
 
   coverage:
     name: Test Coverage

Gemfile

@@ -2,13 +2,19 @@
 
 source "https://rubygems.org"
 
-ruby ">= 3.2.0"
+# Ruby 3.4.0 has ABI incompatibility with nokogiri precompiled binaries.
+# Error: "undefined symbol: ruby_abi_version" when loading nokogiri C extensions.
+# Fixed in Ruby 3.4.1 (verified via testing).
+ruby ">= 3.2.0", "!= 3.4.0"
 
 gem "nokogiri", "~> 1.19.3" # Uses precompiled libxml2 on supported platforms
 gem "open4", "~> 1.3"
 gem "sqlite3", "~> 1.7"
 gem "thread", "~> 0.2"
 
+# drb was removed from Ruby's default gems starting with 3.4.0
+gem "drb", "~> 2.2"
+
 # Legacy Ruby shim libraries for compatibility
 # TODO: These can likely be removed after refactoring parsedate usage to use Date.parse
 gem "rubysl-date", "~> 1.0"

Gemfile.lock

@@ -4,9 +4,11 @@ GEM
     ast (2.4.3)
     diff-lcs (1.6.2)
     docile (1.4.1)
+    drb (2.2.3)
     json (2.19.4)
     language_server-protocol (3.17.0.5)
     lint_roller (1.1.0)
+    mini_portile2 (2.8.9)
     nokogiri (1.19.3-x86_64-linux-gnu)
       racc (~> 1.4)
     open4 (1.3.4)
@@ -62,7 +64,8 @@ GEM
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.13.2)
     simplecov_json_formatter (0.1.4)
-    sqlite3 (1.7.3-x86_64-linux)
+    sqlite3 (1.7.3)
+      mini_portile2 (~> 2.8.0)
     thread (0.2.2)
     unicode-display_width (3.2.0)
       unicode-emoji (~> 4.1)
@@ -72,6 +75,7 @@ PLATFORMS
   x86_64-linux
 
 DEPENDENCIES
+  drb (~> 2.2)
   nokogiri (~> 1.19.3)
   open4 (~> 1.3)
   rake (~> 13.0)

docker/.env

@@ -0,0 +1,20 @@
+# SPDX-License-Identifier: Apache-2.0
+
+# Variables consumed by docker/docker-compose.yml. docker compose reads this
+# file automatically from the compose file's directory. Every variable below
+# also has an inline default in docker-compose.yml, so this file just pins the
+# versions the project builds and tests against.
+
+# Ubuntu base version. Selects which published image variant we pull from the
+# dockerslurmcluster registry (must match a published image tag).
+# Currently supported values: '26.04' or '24.04'
+UBUNTU_VERSION=26.04
+
+# Slurm version. Selects the slurm version suffix for the image tag.
+SLURM_VERSION=25.11.5
+
+# Composite image tag produced from the above:
+#   ghcr.io/noaa-gsl/dockerslurmcluster/slurm-<role>:ubuntu-${UBUNTU_VERSION}-slurm-${SLURM_VERSION}
+#
+# To switch versions for a single invocation without editing this file:
+#   UBUNTU_VERSION=24.04 SLURM_VERSION=25.11.5 docker compose up

docker/docker-compose.yml

@@ -1,102 +1,108 @@
 # SPDX-License-Identifier: Apache-2.0
 
+# Common configuration shared by all compute nodes. Individual node services
+# merge this with `<<: *node-common` and then override the bits that differ
+# (container_name, hostname, environment).
+x-node-common: &node-common
+  image: ghcr.io/noaa-gsl/dockerslurmcluster/slurm-node:ubuntu-${UBUNTU_VERSION:-26.04}-slurm-${SLURM_VERSION:-25.11.5}
+  shm_size: '4g'
+  user: admin
+  volumes:
+    - home-vol:/home/admin
+    - opt-vol:/opt:ro
+    - ..:/home/admin/rocoto
+  depends_on:
+    slurmmaster:
+      condition: service_started
+    slurmfrontend:
+      condition: service_healthy
+  healthcheck:
+    test: ["CMD", "ssh", "-o", "StrictHostKeyChecking=no", "admin@localhost", "hostname"]
+    interval: 2s
+    timeout: 2s
+    retries: 10
+    start_period: 5s
+
 services:
   slurmfrontend:
-        image: ghcr.io/noaa-gsl/dockerslurmcluster/slurm-frontend:latest
-        container_name: frontend
-        hostname: slurmfrontend
-        user: admin
-        volumes:
-                - home-vol:/home/admin
-                - opt-vol:/opt
-                - ..:/home/admin/rocoto
-        ports:
-                - 8888:8888
+    image: ghcr.io/noaa-gsl/dockerslurmcluster/slurm-frontend:ubuntu-${UBUNTU_VERSION:-26.04}-slurm-${SLURM_VERSION:-25.11.5}
+    container_name: rocoto-frontend
+    shm_size: '4g'
+    hostname: slurmfrontend
+    user: admin
+    volumes:
+      - home-vol:/home/admin
+      - opt-vol:/opt
+      - ..:/home/admin/rocoto
+    ports:
+      - 8888:8888
+    healthcheck:
+      test: ["CMD", "test", "-f", "/tmp/ssh-ready"]
+      interval: 5s
+      timeout: 2s
+      retries: 250
+      start_period: 60s
+
   slurmmaster:
-        image: ghcr.io/noaa-gsl/dockerslurmcluster/slurm-master:latest
-        container_name: master
-        hostname: slurmmaster
-        user: admin
-        volumes:
-                - home-vol:/home/admin:nocopy
-                - opt-vol:/opt:ro
-                - ..:/home/admin/rocoto
-        environment:
-                - SLURM_CPUS_ON_NODE=8
-        ports:
-                - 6817:6817
-                - 6818:6818
-                - 6819:6819
+    image: ghcr.io/noaa-gsl/dockerslurmcluster/slurm-master:ubuntu-${UBUNTU_VERSION:-26.04}-slurm-${SLURM_VERSION:-25.11.5}
+    container_name: rocoto-master
+    shm_size: '4g'
+    hostname: slurmmaster
+    user: admin
+    volumes:
+      - home-vol:/home/admin
+      - opt-vol:/opt:ro
+      - ..:/home/admin/rocoto
+    environment:
+      - SLURM_CPUS_ON_NODE=8
+    ports:
+      - 6817:6817
+      - 6818:6818
+      - 6819:6819
+    depends_on:
+      slurmfrontend:
+        condition: service_healthy
+
   slurmnode1:
-        image: ghcr.io/noaa-gsl/dockerslurmcluster/slurm-node:latest
-        container_name: node1
-        hostname: slurmnode1
-        user: admin
-        volumes:
-                - home-vol:/home/admin:nocopy
-                - opt-vol:/opt:ro
-                - ..:/home/admin/rocoto
-        environment:
-                - SLURM_NODENAME=slurmnode1
-                - SLURM_CPUS_ON_NODE=8
-        links:
-                - slurmmaster
+    <<: *node-common
+    container_name: rocoto-node1
+    hostname: slurmnode1
+    environment:
+      - SLURM_NODENAME=slurmnode1
+      - SLURM_CPUS_ON_NODE=8
+
   slurmnode2:
-        image: ghcr.io/noaa-gsl/dockerslurmcluster/slurm-node:latest
-        container_name: node2
-        hostname: slurmnode2
-        user: admin
-        volumes:
-                - home-vol:/home/admin:nocopy
-                - opt-vol:/opt:ro
-                - ..:/home/admin/rocoto
-        environment:
-                - SLURM_NODENAME=slurmnode2
-                - SLURM_CPUS_ON_NODE=8
-        links:
-                - slurmmaster
+    <<: *node-common
+    container_name: rocoto-node2
+    hostname: slurmnode2
+    environment:
+      - SLURM_NODENAME=slurmnode2
+      - SLURM_CPUS_ON_NODE=8
+
   slurmnode3:
-        image: ghcr.io/noaa-gsl/dockerslurmcluster/slurm-node:latest
-        container_name: node3
-        hostname: slurmnode3
-        user: admin
-        volumes:
-                - home-vol:/home/admin:nocopy
-                - opt-vol:/opt:ro
-                - ..:/home/admin/rocoto
-        environment:
-                - SLURM_NODENAME=slurmnode3
-                - SLURM_CPUS_ON_NODE=8
-        links:
-                - slurmmaster
+    <<: *node-common
+    container_name: rocoto-node3
+    hostname: slurmnode3
+    environment:
+      - SLURM_NODENAME=slurmnode3
+      - SLURM_CPUS_ON_NODE=8
+
   slurmnode4:
-        image: ghcr.io/noaa-gsl/dockerslurmcluster/slurm-node:latest
-        container_name: node4
-        hostname: slurmnode4
-        user: admin
-        volumes:
-                - home-vol:/home/admin:nocopy
-                - opt-vol:/opt:ro
-                - ..:/home/admin/rocoto
-        environment:
-                - SLURM_NODENAME=slurmnode4
-                - SLURM_CPUS_ON_NODE=8
-        links:
-                - slurmmaster
+    <<: *node-common
+    container_name: rocoto-node4
+    hostname: slurmnode4
+    environment:
+      - SLURM_NODENAME=slurmnode4
+      - SLURM_CPUS_ON_NODE=8
+
   slurmnode5:
-        image: ghcr.io/noaa-gsl/dockerslurmcluster/slurm-node:latest
-        container_name: node5
-        hostname: slurmnode5
-        user: admin
-        volumes:
-                - home-vol:/home/admin:nocopy
-                - opt-vol:/opt:ro
-                - ..:/home/admin/rocoto
-        environment:
-                - SLURM_NODENAME=slurmnode5
-                - SLURM_CPUS_ON_NODE=8
-        links:
-                - slurmmaster
+    <<: *node-common
+    container_name: rocoto-node5
+    hostname: slurmnode5
+    environment:
+      - SLURM_NODENAME=slurmnode5
+      - SLURM_CPUS_ON_NODE=8
+
 volumes:
-        home-vol:
-        opt-vol:
+  home-vol:
+  opt-vol:

lib/workflowmgr/utilities.rb

@@ -4,6 +4,7 @@
 #
 ##########################################
 require 'English'
+require 'fileutils'
 module WorkflowMgr
   # DRYRUN controls whether workflow operations are executed or just logged.
   # It defaults to 0 (off), but can be overridden via the WORKFLOWMGR_DRYRUN
@@ -139,8 +140,11 @@ def self.stderr(message, level = 0)
     return if message.nil?
     return if message.empty?
 
-    if level <= VERBOSE
-      warn "#{Time.now.strftime('%x %X %Z')} :: #{WORKFLOW_ID} :: #{message}"
+    verbose = defined?(WorkflowMgr::VERBOSE) ? WorkflowMgr::VERBOSE : 0
+    workflow_id = defined?(WorkflowMgr::WORKFLOW_ID) ? WorkflowMgr::WORKFLOW_ID : 'unknown'
+
+    if level <= verbose
+      warn "#{Time.now.strftime('%x %X %Z')} :: #{workflow_id} :: #{message}"
     end
   end
 
@@ -153,8 +157,15 @@ def self.log(message)
     return if message.nil?
     return if message.empty?
 
+    # Get workflow ID, or use 'unknown' if not defined
+    workflow_id = defined?(WorkflowMgr::WORKFLOW_ID) ? WorkflowMgr::WORKFLOW_ID : 'unknown'
+
     # Name of the current log file
-    rocotolog = "#{ENV['HOME']}/.rocoto/#{WorkflowMgr.version}/#{WorkflowMgr::WORKFLOW_ID.sub(/.xml$/, '')}/log"
+    rocotolog = "#{ENV['HOME']}/.rocoto/#{WorkflowMgr.version}/#{workflow_id.sub(/.xml$/, '')}/log"
+
+    # Create the log directory if it doesn't exist
+    logdir = File.dirname(rocotolog)
+    FileUtils.mkdir_p(logdir) unless File.directory?(logdir)
 
     # Logging requires exclusive access to the logs
     # Open the log lock file
@@ -204,17 +215,17 @@ def self.log(message)
 
           # Log the message
           File.open(rocotolog, "a") do |f|
-            f.puts "#{Time.now.strftime('%x %X %Z')} :: #{WorkflowMgr::WORKFLOW_ID} :: #{message}"
+            f.puts "#{Time.now.strftime('%x %X %Z')} :: #{workflow_id} :: #{message}"
           end
         ensure
           # Make sure the lock is released
           lockfile.flock(File::LOCK_UN)
         end
 
       else
-        warn "#{Time.now.strftime('%x %X %Z')} :: #{WorkflowMgr::WORKFLOW_ID} :: " \
+        warn "#{Time.now.strftime('%x %X %Z')} :: #{workflow_id} :: " \
              "WARNING! Could not acquire lock to write log the following message"
-        warn "#{Time.now.strftime('%x %X %Z')} :: #{WorkflowMgr::WORKFLOW_ID} ::          #{message}"
+        warn "#{Time.now.strftime('%x %X %Z')} :: #{workflow_id} ::          #{message}"
       end
     end
   end