Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions platform/mellanox/docker-syncd-mlnx/Dockerfile.j2
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ COPY ["supervisord.conf.j2", "/usr/share/sonic/templates/"]
COPY ["critical_processes", "/etc/supervisor/"]
COPY ["platform_syncd_dump.sh", "/usr/bin/"]
COPY ["phcsync.sh", "/usr/bin/"]
COPY ["phcsync_warm_reboot_gate.py", "/usr/bin/"]

COPY ["files/rdb-cli", "/usr/bin/"]
RUN chmod +x /usr/bin/rdb-cli
Expand Down
123 changes: 123 additions & 0 deletions platform/mellanox/docker-syncd-mlnx/phcsync_warm_reboot_gate.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
#!/usr/bin/env python3
#
# SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES
# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Gate process: runs phcsync.sh as a child and pauses / resumes when warm reboot is detected.
# Uses swsscommon RestartWaiter (event-driven, no DB polling).
#

import os
import signal
import sys
import syslog
import subprocess

try:
from swsscommon.swsscommon import RestartWaiter
except ImportError:
syslog.syslog(syslog.LOG_ERR, "phcsync_warm_reboot_gate: swsscommon not available")
sys.exit(1)

PHCSYNC_SCRIPT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "phcsync.sh")
# 24h timeout match RestartWaiter default, omitting this would yield the same behavior.
# Defined here for readability.
WARM_BOOT_STARTED_TIMEOUT_SEC = 86400 # 24h
WARM_BOOT_DONE_TIMEOUT_SEC = 240 # 4m

_child_proc = None


def log(msg, level=syslog.LOG_INFO):
syslog.syslog(level, msg)


def shutdown(signum, frame):
global _child_proc
if _child_proc is not None and _child_proc.pid is not None:
try:
_child_proc.terminate()
_child_proc.wait(timeout=5)
except (ProcessLookupError, subprocess.TimeoutExpired):
try:
_child_proc.kill()
except ProcessLookupError:
pass

# Exit with 128 + signal number to indicate this program terminated due to a signal (SIGINT/SIGTERM).
sys.exit(128 + (signum if signum in (signal.SIGINT, signal.SIGTERM) else 0))


def main():
global _child_proc
syslog.openlog("phcsync-warm-reboot-gate", syslog.LOG_PID)

if not os.path.isfile(PHCSYNC_SCRIPT) or not os.access(PHCSYNC_SCRIPT, os.X_OK):
log("phcsync script not found or not executable: %s" % PHCSYNC_SCRIPT, syslog.LOG_ERR)
return 1

_child_proc = None

signal.signal(signal.SIGTERM, shutdown)
signal.signal(signal.SIGINT, shutdown)

try:
_child_proc = subprocess.Popen(
[PHCSYNC_SCRIPT],
# Run child in a new process group to avoid receiving signals sent to the parent
preexec_fn=os.setpgrp if hasattr(os, "setpgrp") else None,
)
except Exception as e:
log("Failed to start %s: %s" % (PHCSYNC_SCRIPT, e), syslog.LOG_ERR)
return 1

log("Started phcsync (pid %d), listening for warm reboot" % _child_proc.pid)

while True:
# Wait for warm boot to start
if not RestartWaiter.waitWarmBootStarted(maxWaitSec=WARM_BOOT_STARTED_TIMEOUT_SEC):
log("waitWarmBootStarted timed out, continuing loop", syslog.LOG_DEBUG)
continue

# If warm boot started, check if phcsync process is still running
pid = _child_proc.pid
if pid is None:
break
try:
# Check if phcsync process is still running - use WNOHANG to avoid blocking
_pid, _ = os.waitpid(pid, os.WNOHANG)
if _pid != 0:
log("phcsync exited (pid=%d), gate exiting" % pid, syslog.LOG_WARNING)
return 2
except ChildProcessError:
break

# If _pid =0 phcsync is still running, pause it
log("Warm reboot started, pausing phcsync")
try:
os.kill(pid, signal.SIGSTOP)
except ProcessLookupError:
log("phcsync process gone, exiting", syslog.LOG_WARNING)
return 2

# Wait for warm boot to complete.
if not RestartWaiter.waitWarmBootDone(maxWaitSec=WARM_BOOT_DONE_TIMEOUT_SEC):
# If waitWarmBootDone timed out, resume phcsync and check again for warm boot (via RestartWaiter.waitWarmBootStarted)
log("waitWarmBootDone timed out, resuming phcsync anyway", syslog.LOG_WARNING)

# Resume phcsync
try:
os.kill(pid, signal.SIGCONT)
except ProcessLookupError:
log("phcsync process gone while trying to resume, exiting", syslog.LOG_WARNING)
return 2
log("Warm reboot done or timed out, resumed phcsync")

# The main loop is expected to run indefinitely, reaching here means unexpected termination.
log("Unexpected termination, exiting", syslog.LOG_WARNING)
return 2


if __name__ == "__main__":
sys.exit(main())
12 changes: 7 additions & 5 deletions platform/mellanox/docker-syncd-mlnx/supervisord.conf.j2
Original file line number Diff line number Diff line change
Expand Up @@ -46,12 +46,14 @@ dependent_startup_wait_for=rsyslogd:running
environment=ASAN_OPTIONS="log_path=/var/log/asan/syncd-asan.log{{ asan_extra_options }}"
{% endif %}

[program:phcsync]
command=/usr/bin/phcsync.sh
[program:phcsync-gate]
command=/usr/bin/phcsync_warm_reboot_gate.py
priority=3
autostart=false
autorestart=false
stdout_logfile=syslog
stderr_logfile=syslog
autorestart=unexpected
stdout_logfile=NONE
stdout_syslog=true
stderr_logfile=NONE
stderr_syslog=true
dependent_startup=true
dependent_startup_wait_for=rsyslogd:running
Loading