Skip to content

Commit 95f50c4

Browse files
Pigbibicodex
Pigbibi
and
codex
committed
fix(ops): order gateway activity markers
Co-Authored-By: Codex <noreply@openai.com>
1 parent c78f4f6 commit 95f50c4

3 files changed

Lines changed: 93 additions & 87 deletions

File tree

‎scripts/classify_ib_gateway_activity.awk‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
BEGIN {
2+
timestamp_pattern = "^[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9][T ][0-9][0-9]:[0-9][0-9]:[0-9][0-9]$"
3+
latest_timestamp = ""
4+
latest_state = ""
5+
}
6+
7+
{
8+
timestamp = substr($0, 1, 19)
9+
if (timestamp !~ timestamp_pattern) {
10+
next
11+
}
12+
13+
gsub("T", " ", timestamp)
14+
if (cutoff_timestamp != "" && timestamp < cutoff_timestamp) {
15+
next
16+
}
17+
18+
state = ""
19+
if ($0 ~ progress_regex) {
20+
state = "progress"
21+
}
22+
if ($0 ~ terminal_regex) {
23+
state = "terminal"
24+
}
25+
26+
if (state != "" && (timestamp > latest_timestamp ||
27+
(timestamp == latest_timestamp && state == "terminal"))) {
28+
latest_timestamp = timestamp
29+
latest_state = state
30+
}
31+
}
32+
33+
END {
34+
if (latest_state != "") {
35+
print latest_timestamp "\t" latest_state
36+
}
37+
}

‎scripts/recover_ib_gateway_ready.sh‎

Lines changed: 33 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ progress_extensions="${IB_GATEWAY_RECOVERY_PROGRESS_EXTENSIONS:-2}"
1616
progress_window_seconds="${IB_GATEWAY_RECOVERY_PROGRESS_WINDOW_SECONDS:-420}"
1717
progress_regex="${IB_GATEWAY_RECOVERY_PROGRESS_REGEX:-IBC: (Starting Gateway|Login attempt|Second Factor Authentication|Login has completed|Configuration tasks completed|Found Gateway main window|Getting config dialog|Getting main window)|Authentication window found|Auto-fill submitted|Passed token authentication|Authentication completed|Security code:}"
1818
terminal_regex="${IB_GATEWAY_RECOVERY_TERMINAL_REGEX:-Connection reset by peer|Server disconnected|IBC: .*(Authentication|Login).*(timed out|timeout|failed)|IBC: .*(timed out|timeout).*(Authentication|Login)}"
19+
activity_classifier="${script_dir}/classify_ib_gateway_activity.awk"
1920
lock_file="${IB_GATEWAY_RECOVERY_LOCK_FILE:-/var/lock/ib_gateway_recovery.lock}"
2021
lock_wait_seconds="${IB_GATEWAY_RECOVERY_LOCK_WAIT_SECONDS:-900}"
2122

@@ -42,98 +43,54 @@ wait_for_ready() {
4243
bash "${script_dir}/wait_for_ib_gateway_ready.sh" "${gateway_mode}"
4344
}
4445

45-
gateway_recently_progressing_from_docker_logs() {
46-
docker logs --since "${progress_window_seconds}s" "${container_name}" 2>&1 \
47-
| grep -Eiq "${progress_regex}"
46+
gateway_recent_activity_from_docker_logs() {
47+
docker logs --timestamps --since "${progress_window_seconds}s" "${container_name}" 2>&1 \
48+
| awk -v progress_regex="${progress_regex}" -v terminal_regex="${terminal_regex}" \
49+
-f "${activity_classifier}"
4850
}
4951

50-
gateway_recently_progressing_from_file_logs() {
51-
docker exec "${container_name}" sh -s -- "${progress_window_seconds}" "${progress_regex}" <<'SH'
52-
set -eu
53-
54-
progress_window_seconds="$1"
55-
progress_regex="$2"
56-
now="$(date +%s)"
57-
cutoff_timestamp="$(date -u -d "@$((now - progress_window_seconds))" "+%Y-%m-%d %H:%M:%S")"
58-
59-
for log_path in /home/ibgateway/Jts/launcher.log /home/ibgateway/2fa.log; do
60-
if [ ! -f "${log_path}" ]; then
61-
continue
62-
fi
63-
64-
log_mtime="$(stat -c %Y "${log_path}" 2>/dev/null || echo 0)"
65-
if [ $((now - log_mtime)) -le "${progress_window_seconds}" ]; then
66-
tail -n 400 "${log_path}" 2>/dev/null \
67-
| awk -v cutoff_timestamp="${cutoff_timestamp}" -v progress_regex="${progress_regex}" '
68-
substr($0, 1, 19) >= cutoff_timestamp && $0 ~ progress_regex { found = 1 }
69-
END { exit found ? 0 : 1 }
70-
' && exit 0
71-
fi
72-
done
73-
74-
exit 1
75-
SH
76-
}
77-
78-
gateway_recently_progressing() {
79-
gateway_recently_progressing_from_docker_logs || gateway_recently_progressing_from_file_logs
80-
}
81-
82-
gateway_recently_terminal_from_docker_logs() {
83-
docker logs --since "${progress_window_seconds}s" "${container_name}" 2>&1 \
84-
| grep -Eiq "${terminal_regex}"
85-
}
86-
87-
gateway_recently_terminal_from_file_logs() {
88-
docker exec "${container_name}" sh -s -- "${progress_window_seconds}" "${terminal_regex}" <<'SH'
89-
set -eu
90-
91-
progress_window_seconds="$1"
92-
terminal_regex="$2"
93-
now="$(date +%s)"
94-
cutoff_timestamp="$(date -u -d "@$((now - progress_window_seconds))" "+%Y-%m-%d %H:%M:%S")"
95-
96-
for log_path in /home/ibgateway/Jts/launcher.log /home/ibgateway/2fa.log; do
97-
if [ ! -f "${log_path}" ]; then
98-
continue
99-
fi
100-
101-
log_mtime="$(stat -c %Y "${log_path}" 2>/dev/null || echo 0)"
102-
if [ $((now - log_mtime)) -le "${progress_window_seconds}" ]; then
103-
tail -n 400 "${log_path}" 2>/dev/null \
104-
| awk -v cutoff_timestamp="${cutoff_timestamp}" -v terminal_regex="${terminal_regex}" '
105-
substr($0, 1, 19) >= cutoff_timestamp && $0 ~ terminal_regex { found = 1 }
106-
END { exit found ? 0 : 1 }
107-
' && exit 0
108-
fi
109-
done
110-
111-
exit 1
112-
SH
52+
gateway_recent_activity_from_file_logs() {
53+
local cutoff_timestamp
54+
local log_path
55+
local now
56+
now="$(date +%s)"
57+
cutoff_timestamp="$(date -u -d "@$((now - progress_window_seconds))" "+%Y-%m-%d %H:%M:%S")"
58+
59+
for log_path in /home/ibgateway/Jts/launcher.log /home/ibgateway/2fa.log; do
60+
{ docker exec "${container_name}" tail -n 400 "${log_path}" 2>/dev/null || true; } \
61+
| awk -v cutoff_timestamp="${cutoff_timestamp}" \
62+
-v progress_regex="${progress_regex}" -v terminal_regex="${terminal_regex}" \
63+
-f "${activity_classifier}"
64+
done
11365
}
11466

115-
gateway_recently_terminal() {
116-
gateway_recently_terminal_from_docker_logs || gateway_recently_terminal_from_file_logs
67+
gateway_recent_activity() {
68+
{
69+
gateway_recent_activity_from_docker_logs || true
70+
gateway_recent_activity_from_file_logs || true
71+
} | LC_ALL=C sort | tail -n 1 | cut -f 2
11772
}
11873

11974
wait_for_ready_with_progress() {
12075
local timeout_seconds="$1"
12176
local stage="$2"
12277
local extension=0
78+
local activity
12379

12480
if wait_for_ready "${timeout_seconds}"; then
12581
return 0
12682
fi
12783

12884
while [ "${extension}" -lt "${progress_extensions}" ]; do
129-
if gateway_recently_terminal; then
130-
echo "Recent terminal IB gateway authentication failure detected after ${stage} wait; skipping progress extension." >&2
131-
return 1
132-
fi
133-
134-
if ! gateway_recently_progressing; then
135-
return 1
136-
fi
85+
activity="$(gateway_recent_activity)"
86+
case "${activity}" in
87+
terminal)
88+
echo "Recent terminal IB gateway authentication failure detected after ${stage} wait; skipping progress extension." >&2
89+
return 1
90+
;;
91+
progress) ;;
92+
*) return 1 ;;
93+
esac
13794

13895
extension=$((extension + 1))
13996
echo "Recent IB gateway login/config progress detected after ${stage} wait; extending readiness wait (${extension}/${progress_extensions}) by ${progress_wait_seconds}s before external recovery." >&2

‎tests/test_gateway_recovery_scripts.sh‎

Lines changed: 23 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,15 @@ set -euo pipefail
33

44
repo_dir="$(cd "$(dirname "$0")/.." && pwd)"
55
recover_script="$repo_dir/scripts/recover_ib_gateway_ready.sh"
6+
activity_classifier="$repo_dir/scripts/classify_ib_gateway_activity.awk"
67
twofa_bot="$repo_dir/2fa_bot.py"
78
swap_script="$repo_dir/scripts/ensure_host_swap.sh"
89
daily_restart_script="$repo_dir/scripts/restart_ib_gateway_daily.sh"
910
health_watcher_script="$repo_dir/scripts/install_gateway_health_watcher.sh"
1011
unit_helper_script="$repo_dir/scripts/ibkr_gateway_units.sh"
1112

1213
test -f "$recover_script"
14+
test -f "$activity_classifier"
1315
test -f "$swap_script"
1416
test -f "$daily_restart_script"
1517
test -f "$health_watcher_script"
@@ -30,7 +32,7 @@ grep -Fq 'Passed token authentication' "$recover_script"
3032
grep -Fq 'Authentication completed' "$recover_script"
3133
grep -Fq 'Connection reset by peer' "$recover_script"
3234
grep -Fq 'Server disconnected' "$recover_script"
33-
grep -Fq 'gateway_recently_terminal()' "$recover_script"
35+
grep -Fq 'gateway_recent_activity()' "$recover_script"
3436
grep -Fq 'Recent terminal IB gateway authentication failure detected' "$recover_script"
3537
if grep -Fq 'Dismissing post-login dialog' "$recover_script" "$twofa_bot"; then
3638
echo 'Ambiguous dismiss-dialog text must not be treated as recovery progress' >&2
@@ -48,21 +50,31 @@ fi
4850
printf '%s\n' 'Connection reset by peer' | grep -Eq "$default_terminal_regex"
4951
printf '%s\n' 'Server disconnected' | grep -Eq "$default_terminal_regex"
5052
printf '%s\n' 'IBC: Login attempt timed out' | grep -Eq "$default_terminal_regex"
51-
grep -Fq 'gateway_recently_progressing()' "$recover_script"
52-
grep -Fq 'gateway_recently_progressing_from_docker_logs()' "$recover_script"
53-
grep -Fq 'gateway_recently_progressing_from_file_logs()' "$recover_script"
53+
newer_progress="$(printf '%s\n' \
54+
'2026-07-15 16:00:01 Server disconnected' \
55+
'2026-07-15 16:00:02 IBC: Login attempt' \
56+
| awk -v cutoff_timestamp='2026-07-15 16:00:00' -v progress_regex="$default_progress_regex" -v terminal_regex="$default_terminal_regex" -f "$activity_classifier")"
57+
test "$newer_progress" = $'2026-07-15 16:00:02\tprogress'
58+
newer_terminal="$(printf '%s\n' \
59+
'2026-07-15 16:00:01 IBC: Login attempt' \
60+
'2026-07-15 16:00:02 Server disconnected' \
61+
| awk -v cutoff_timestamp='2026-07-15 16:00:00' -v progress_regex="$default_progress_regex" -v terminal_regex="$default_terminal_regex" -f "$activity_classifier")"
62+
test "$newer_terminal" = $'2026-07-15 16:00:02\tterminal'
63+
untimestamped_terminal="$(printf '%s\n' \
64+
'2026-07-15 15:59:59 IBC: Login attempt' \
65+
'Server disconnected' \
66+
| awk -v cutoff_timestamp='2026-07-15 16:00:00' -v progress_regex="$default_progress_regex" -v terminal_regex="$default_terminal_regex" -f "$activity_classifier")"
67+
test -z "$untimestamped_terminal"
68+
grep -Fq 'gateway_recent_activity_from_docker_logs()' "$recover_script"
69+
grep -Fq 'gateway_recent_activity_from_file_logs()' "$recover_script"
70+
grep -Fq 'docker logs --timestamps --since' "$recover_script"
5471
grep -Fq '/home/ibgateway/Jts/launcher.log' "$recover_script"
5572
grep -Fq '/home/ibgateway/2fa.log' "$recover_script"
56-
grep -Fq 'stat -c %Y "${log_path}"' "$recover_script"
5773
grep -Fq 'cutoff_timestamp="$(date -u -d "@$((now - progress_window_seconds))" "+%Y-%m-%d %H:%M:%S")"' "$recover_script"
58-
grep -Fq 'substr($0, 1, 19) >= cutoff_timestamp && $0 ~ progress_regex' "$recover_script"
74+
grep -Fq 'activity="$(gateway_recent_activity)"' "$recover_script"
5975
grep -Fq 'wait_for_ready_with_progress()' "$recover_script"
6076
grep -Fq 'Recent IB gateway login/config progress detected' "$recover_script"
61-
terminal_check_line="$(grep -n 'if gateway_recently_terminal; then' "$recover_script" | head -n 1 | cut -d: -f1)"
62-
progress_check_line="$(grep -n 'if ! gateway_recently_progressing; then' "$recover_script" | head -n 1 | cut -d: -f1)"
63-
test -n "$terminal_check_line"
64-
test -n "$progress_check_line"
65-
test "$terminal_check_line" -lt "$progress_check_line"
77+
grep -Fq 'case "${activity}" in' "$recover_script"
6678
grep -Fq 'IB_GATEWAY_RECOVERY_LOCK_FILE:-/var/lock/ib_gateway_recovery.lock' "$recover_script"
6779
grep -Fq 'IB_GATEWAY_RECOVERY_LOCK_WAIT_SECONDS:-900' "$recover_script"
6880
grep -Fq 'flock -n 9' "$recover_script"

0 commit comments

Comments
 (0)