Skip to content

Commit 6ca632f

Browse files
awksedgreepclaude
andcommitted
production gate: fixture base moves with the clock, not a constant
The Rust gate port hardcoded BASE_SECONDS = 2026-08-02T00:00:00Z. The metrics server prunes raw data at wall-clock now - 7 days, so exactly seven days later the fixture's points aged out of retention mid-run and the gate failed on every commit — including v0.4.0, which had passed it on 2026-08-08 — with the same deterministic "overlap snapshot 3456 outside admission window [6848, 6848]" on CI and locally. Bisecting code was useless by construction; the failure traveled with the date. The base is now the most recent UTC midnight via OnceLock: granule alignment and per-run determinism are preserved, and fixture data is at most 24h old against the 7-day window. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1 parent aceed83 commit 6ca632f

2 files changed

Lines changed: 47 additions & 14 deletions

File tree

CHANGELOG.md

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,13 @@ lockfile refresh and the documentation the contract gate flagged:
2727
three 0.6 releases, and `timeless_pins` was registered without an inventory
2828
row. `docs/RELEASING.md` now records the full pre-tag checklist.
2929

30+
Also defuses a time bomb in the production fault gate: fixture timestamps
31+
were offsets from a hardcoded 2026-08-02 base while the metrics server
32+
prunes at wall-clock now minus 7 days, so from 2026-08-09 the gate failed
33+
everywhere with "overlap snapshot outside admission window" on any commit —
34+
including tags that had passed it before. The base is now the most recent
35+
UTC midnight (alignment and determinism preserved; data at most 24h old).
36+
3037
### Fixed (as unreleased `0.6.3`)
3138

3239
### Fixed

tools/query-harness/src/production.rs

Lines changed: 40 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ use std::process::{Child, Command, Stdio};
1010
use std::sync::atomic::{AtomicBool, Ordering};
1111
use std::sync::{Arc, Mutex, MutexGuard};
1212
use std::thread::{self, JoinHandle};
13-
use std::time::{Duration, Instant};
13+
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
1414

1515
use anyhow::{anyhow, bail, Context, Result};
1616
use chrono::{SecondsFormat, Utc};
@@ -22,10 +22,35 @@ use serde_json::{json, Map, Number, Value};
2222
use tempfile::TempDir;
2323
use wait_timeout::ChildExt;
2424

25-
const BASE_SECONDS: u64 = 1_785_628_800;
25+
/// Fixture timestamps are offsets from the most recent UTC midnight, NOT a
26+
/// constant. The original constant base (2026-08-02) was a time bomb: the
27+
/// metrics server prunes raw data at `now - 7 days` of wall-clock time, so
28+
/// exactly seven days after the constant was written the gate started
29+
/// failing everywhere ("overlap snapshot outside admission window") with no
30+
/// code change — the fixture's points were simply aging out of retention
31+
/// mid-run. A midnight-aligned dynamic base keeps rollup-granule alignment
32+
/// and per-run determinism while the data stays at most 24h old.
33+
fn base_seconds() -> u64 {
34+
use std::sync::OnceLock;
35+
static BASE: OnceLock<u64> = OnceLock::new();
36+
*BASE.get_or_init(|| {
37+
let now = SystemTime::now()
38+
.duration_since(UNIX_EPOCH)
39+
.expect("system time precedes Unix epoch")
40+
.as_secs();
41+
now / 86_400 * 86_400
42+
})
43+
}
44+
2645
const ORDINALS_PER_SECOND: u64 = 256;
27-
const BASE_MILLISECONDS: u64 = BASE_SECONDS * 1_000;
28-
const BASE_NANOSECONDS: u64 = BASE_SECONDS * 1_000_000_000;
46+
47+
fn base_milliseconds() -> u64 {
48+
base_seconds() * 1_000
49+
}
50+
51+
fn base_nanoseconds() -> u64 {
52+
base_seconds() * 1_000_000_000
53+
}
2954
const MIN_RELEASE_SECONDS_PER_SIGNAL: f64 = 2.0 * 60.0 * 60.0;
3055
const RELEASE_AGGREGATE_SIGNAL_HOURS: f64 = 8.0;
3156
const DEFAULT_RELEASE_SECONDS: f64 = RELEASE_AGGREGATE_SIGNAL_HOURS * 60.0 * 60.0 / 3.0;
@@ -645,7 +670,7 @@ fn metrics_body(start: u64, count: usize) -> Vec<u8> {
645670
for ordinal in start..start + count as u64 {
646671
grouped[ordinal as usize % 4].push((
647672
ordinal as f64 + 0.5,
648-
BASE_MILLISECONDS + ordinal * 1_000 / ORDINALS_PER_SECOND,
673+
base_milliseconds() + ordinal * 1_000 / ORDINALS_PER_SECOND,
649674
));
650675
}
651676
let mut output = String::new();
@@ -674,7 +699,7 @@ fn logs_body(start: u64, count: usize) -> Vec<u8> {
674699
for ordinal in start..start + count as u64 {
675700
output.push_str(
676701
&serde_json::to_string(&json!({
677-
"_time": BASE_SECONDS + ordinal / ORDINALS_PER_SECOND,
702+
"_time": base_seconds() + ordinal / ORDINALS_PER_SECOND,
678703
"_msg": format!("release-gate-{ordinal}"),
679704
"level": levels[ordinal as usize % levels.len()],
680705
"service": "release-gate",
@@ -697,7 +722,7 @@ fn traces_body(start: u64, count: usize) -> Vec<u8> {
697722
.map(|ordinal| {
698723
let trace_number = ordinal / 4 + 1;
699724
let root_ordinal = ordinal / 4 * 4;
700-
let start_ns = BASE_NANOSECONDS + ordinal * 1_000_000_000 / ORDINALS_PER_SECOND;
725+
let start_ns = base_nanoseconds() + ordinal * 1_000_000_000 / ORDINALS_PER_SECOND;
701726
json!({
702727
"traceId": format!("{trace_number:032x}"),
703728
"spanId": format!("{:016x}", ordinal + 1),
@@ -816,16 +841,16 @@ fn result_rows(result: &HttpResult) -> Result<u64> {
816841

817842
fn metrics_query(client: &Client, state: &SignalState, shape: &str) -> Result<(f64, u64, u64)> {
818843
let newest = state.data()?.next_ordinal.saturating_sub(1);
819-
let newest_seconds = BASE_SECONDS + newest / ORDINALS_PER_SECOND;
820-
let from_seconds = BASE_SECONDS.max(newest_seconds.saturating_sub(300));
844+
let newest_seconds = base_seconds() + newest / ORDINALS_PER_SECOND;
845+
let from_seconds = base_seconds().max(newest_seconds.saturating_sub(300));
821846
let path = match shape {
822847
"exact_latest" => "/api/v1/query?metric=release_gate_metric&host=host-0".to_owned(),
823848
"narrow_range" => format!(
824849
"/api/v1/query_range?metric=release_gate_metric&host=host-0&from={from_seconds}&to={newest_seconds}&step=10&aggregate=avg"
825850
),
826851
"wide_range" => format!(
827852
"/api/v1/query_range?metric=release_gate_metric&from={}&to={newest_seconds}&step=10&aggregate=avg",
828-
BASE_SECONDS.max(newest_seconds.saturating_sub(60))
853+
base_seconds().max(newest_seconds.saturating_sub(60))
829854
),
830855
"scalar_avg" => format!(
831856
"/api/v1/query_range?metric=release_gate_metric&host=host-1&from={from_seconds}&to={newest_seconds}&step=300&aggregate=avg"
@@ -857,13 +882,13 @@ fn metrics_query(client: &Client, state: &SignalState, shape: &str) -> Result<(f
857882

858883
fn logs_query(client: &Client, state: &SignalState, shape: &str) -> Result<(f64, u64, u64)> {
859884
let newest = state.data()?.next_ordinal.saturating_sub(1);
860-
let newest_seconds = BASE_SECONDS + newest / ORDINALS_PER_SECOND;
885+
let newest_seconds = base_seconds() + newest / ORDINALS_PER_SECOND;
861886
let (method, path, body, headers) = match shape {
862887
"exact" => (Method::GET, "/select/logsql/query?message=release-gate-0&limit=1&order=asc".to_owned(), None, vec![]),
863888
"narrow" => (Method::GET, "/select/logsql/query?level=error&service=release-gate&limit=100&order=desc".to_owned(), None, vec![]),
864889
"wide" => (Method::GET, format!(
865890
"/select/logsql/query?service=release-gate&limit=1000&order=desc&start={}&end={newest_seconds}",
866-
BASE_SECONDS.max(newest_seconds.saturating_sub(4_000))
891+
base_seconds().max(newest_seconds.saturating_sub(4_000))
867892
), None, vec![]),
868893
"scalar_count" => (Method::POST, "/select/logsql/query".to_owned(), Some(b"query=level%3Aerror+%7C+stats+count%28*%29".to_vec()), vec![("content-type", "application/x-www-form-urlencoded")]),
869894
"discovery" => (Method::GET, "/select/logsql/field_values?field=host&service=release-gate&limit=10".to_owned(), None, vec![]),
@@ -951,8 +976,9 @@ fn query_once(client: &Client, state: &SignalState, shape: &str) -> Result<()> {
951976
fn semantic_oracle(client: &Client, target: Target) -> Result<()> {
952977
match target.signal {
953978
Signal::Metrics => {
979+
let base = base_seconds();
954980
let path = format!(
955-
"/api/v1/export?metric=release_gate_metric&host=host-0&from={BASE_SECONDS}&to={BASE_SECONDS}"
981+
"/api/v1/export?metric=release_gate_metric&host=host-0&from={base}&to={base}"
956982
);
957983
let result = require_status(
958984
http_request(
@@ -981,7 +1007,7 @@ fn semantic_oracle(client: &Client, target: Target) -> Result<()> {
9811007
.and_then(Value::as_array);
9821008
let has_value = values.is_some_and(|values| values.contains(&json!(0.5)));
9831009
let has_timestamp =
984-
timestamps.is_some_and(|values| values.contains(&json!(BASE_MILLISECONDS)));
1010+
timestamps.is_some_and(|values| values.contains(&json!(base_milliseconds())));
9851011
if rows.len() != 1
9861012
|| !has_value
9871013
|| !has_timestamp

0 commit comments

Comments
 (0)