From 226a4eebe4548b7601030d02c2b02a00ccc955bb Mon Sep 17 00:00:00 2001 From: Nipun Batra Date: Fri, 17 Jul 2026 18:24:39 +0530 Subject: [PATCH] Generate one rank per evaluation cohort --- README.md | 16 +-- index.html | 20 +--- leaderboard.csv | 36 +++--- leaderboard.json | 223 ++++++++++++++++++++++++++++++++++- src/nilmbench/leaderboard.py | 38 +++++- tests/test_leaderboard.py | 37 ++++++ tests/test_site.py | 13 +- 7 files changed, 334 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index 4ce646d..794d09d 100644 --- a/README.md +++ b/README.md @@ -181,13 +181,15 @@ SHA-256 so consumers can reject a partially updated artifact pair. Every aggregate is separated by task/config revision, model revision, runner revision, container digest, hardware, resolution, appliance, target-data access, smoke/full scope, and a digest of every protocol override. Context length, -epochs, and sample limits remain visible in the generated table. -The website's rank groups use a separate comparison-protocol digest covering -effective context length and epochs, tuning method/trial budget/selection -protocol, and the immutable runner, contrib, container, and hardware tuple. It -intentionally excludes model-specific selected parameters and the study digest, -so different models remain comparable and evaluation seeds from one frozen -study can still aggregate. A corrected +epochs, sample limits, tuning records, and runtime provenance remain visible in +the generated artifacts. + +Public rank is generated once, in Python, within an evaluation cohort defined by +task/config revision, profile, resolution, appliance, sample limit, target-data +access, and smoke/full scope. Architecture choices such as context length and +epoch count, and provenance changes such as a later container build, do not +reset rank. The complete comparison-protocol digest still records those details +for audit and exact reproduction. A corrected full run becomes `full-verified`, and a smoke run becomes `smoke-verified`, only after the required seeds 10, 20, and 42 pass source, container, and dataset provenance checks. Incomplete clean smoke matrices are labelled `smoke-partial`. diff --git a/index.html b/index.html index b2ab74b..368de27 100644 --- a/index.html +++ b/index.html @@ -925,7 +925,7 @@

Built to become NILM's ImageNet

Loading immutable result bundles…
- + @@ -1558,12 +1558,6 @@

Cite NILMBench2026

function memoryInMiB(value) { return value == null ? '—' : `${(Number(value) / 1048576).toFixed(1)} MiB`; } -function comparisonKey(entry) { - return [ - entry.comparison_protocol_sha256, - entry.appliance, - ].join('|'); -} async function loadLiveLeaderboard() { const body = document.getElementById('live-leaderboard-body'); const summary = document.getElementById('live-leaderboard-summary'); @@ -1578,9 +1572,10 @@

Cite NILMBench2026

return; } const statusOrder = {'full-verified': 0, 'smoke-verified': 1, 'smoke-partial': 2, 'candidate': 3, 'smoke-unverified': 4}; - entries.sort((a,b)=>comparisonKey(a).localeCompare(comparisonKey(b)) || - (statusOrder[a.status]??9)-(statusOrder[b.status]??9) || a.mae_mean-b.mae_mean); - const ranks = new Map(); + entries.sort((a,b)=>a.ranking_protocol_sha256.localeCompare(b.ranking_protocol_sha256) || + (statusOrder[a.status]??9)-(statusOrder[b.status]??9) || + (a.rank??Number.MAX_SAFE_INTEGER)-(b.rank??Number.MAX_SAFE_INTEGER) || + a.mae_mean-b.mae_mean); body.innerHTML = entries.map((entry) => { const protocol = [ entry.sequence_length ? `seq ${entry.sequence_length}` : null, @@ -1589,10 +1584,7 @@

Cite NILMBench2026

].filter(Boolean).join(' / '); const task = `${entry.task} / ${entry.appliance} / ${entry.sample_period}s` + (protocol ? ` / ${protocol}` : ''); - const key = comparisonKey(entry); - const isVerified = entry.status === 'full-verified' || entry.status === 'smoke-verified'; - const rank = isVerified ? (ranks.get(key) ?? 0) + 1 : null; - if (rank != null) ranks.set(key, rank); + const rank = Number.isInteger(entry.rank) ? entry.rank : null; const cls = rank === 1 ? ' class="gold"' : ''; return `` + `` + diff --git a/leaderboard.csv b/leaderboard.csv index 322c063..9b93cce 100644 --- a/leaderboard.csv +++ b/leaderboard.csv @@ -1,18 +1,18 @@ -task,family,profile,model,model_family,model_params_sha256,tuning_study_digest,model_git_sha,runner_git_sha,container_digest,hardware,sample_period,sequence_length,epochs,max_samples_per_window,protocol_overrides_sha256,comparison_protocol_sha256,appliance,target_data_access,scope,status,seeds,run_count,mae_mean,mae_std,f1_mean,f1_std,elapsed_seconds_mean,elapsed_seconds_std,trainable_parameters_mean,trainable_parameters_std,peak_accelerator_memory_bytes_mean,peak_accelerator_memory_bytes_std -corrected-t1-redd,T1,corrected,NILMFormer,transformer,9d8de5f72b1010780ee736900644210cd60be6f1e27f64680ed20eaa30c158b6,,825740b39bcd44b3f4bfaf146f4c0d944843b131,83fb39e57d50ac433314e880176fef187997d5b3,sha256:dd977962c2e0d72e2d923f8f5c3e92e538f67a00495e545c2f22571001872e91,NVIDIA A100-SXM4-80GB,60,299,3,3960,7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056,68326c07a5ae30495f929d97858f5b49e5cfdeaddf78334066d74b90417f5d9e,fridge,not_applicable,smoke,smoke-verified,10;20;42,3,48.08586044774824,0.6277903001830687,0.6281731157580089,0.006296768732152522,57.085627023984365,0.9263972298357254,383283.0,0.0,4020728832.0,0.0 -corrected-t1-redd,T1,corrected,Seq2Point,convolutional,9d8de5f72b1010780ee736900644210cd60be6f1e27f64680ed20eaa30c158b6,,825740b39bcd44b3f4bfaf146f4c0d944843b131,83fb39e57d50ac433314e880176fef187997d5b3,sha256:dd977962c2e0d72e2d923f8f5c3e92e538f67a00495e545c2f22571001872e91,NVIDIA A100-SXM4-80GB,60,299,3,3960,7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056,68326c07a5ae30495f929d97858f5b49e5cfdeaddf78334066d74b90417f5d9e,fridge,not_applicable,smoke,smoke-verified,10;20;42,3,49.13913551508602,1.368821477825844,0.738922168844469,0.02550131120698661,11.937222305607671,0.9718365327389646,13863449.0,0.0,724345856.0,0.0 -corrected-t1-redd,T1,corrected,DAE,autoencoder,07f5562c93a226b28c0f287f72c181f6a7d322c0939fb2d5ca379e6449fc5445,,825740b39bcd44b3f4bfaf146f4c0d944843b131,83fb39e57d50ac433314e880176fef187997d5b3,sha256:dd977962c2e0d72e2d923f8f5c3e92e538f67a00495e545c2f22571001872e91,NVIDIA A100-SXM4-80GB,60,99,3,3960,fe0a640391067d0d8eef3655b630a41151a1bdf4cbff179ce47626525764d28a,81fbe9a0cd8d218ec6bbef16b85d38833c4263704915fb7564a3a69ae878ad0d,fridge,not_applicable,smoke,smoke-verified,10;20;42,3,49.81314141919794,1.2750812946872356,0.7096876676310145,0.011508582889399363,11.623419809659632,1.069115608188479,831801.0,0.0,83911168.0,0.0 -corrected-t1-redd,T1,corrected,WindowGRU,recurrent,9d8de5f72b1010780ee736900644210cd60be6f1e27f64680ed20eaa30c158b6,,825740b39bcd44b3f4bfaf146f4c0d944843b131,83fb39e57d50ac433314e880176fef187997d5b3,sha256:dd977962c2e0d72e2d923f8f5c3e92e538f67a00495e545c2f22571001872e91,NVIDIA A100-SXM4-80GB,60,299,3,3960,7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056,68326c07a5ae30495f929d97858f5b49e5cfdeaddf78334066d74b90417f5d9e,fridge,not_applicable,smoke,smoke-verified,10;20;42,3,50.47948637432652,1.3556019544833675,0.6971819197814662,0.007023031532984379,11.894063677988015,0.13453522512836433,427345.0,0.0,2746802688.0,0.0 -corrected-t1-redd,T1,corrected,Seq2Seq,recurrent,9d8de5f72b1010780ee736900644210cd60be6f1e27f64680ed20eaa30c158b6,,825740b39bcd44b3f4bfaf146f4c0d944843b131,83fb39e57d50ac433314e880176fef187997d5b3,sha256:dd977962c2e0d72e2d923f8f5c3e92e538f67a00495e545c2f22571001872e91,NVIDIA A100-SXM4-80GB,60,299,3,3960,7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056,68326c07a5ae30495f929d97858f5b49e5cfdeaddf78334066d74b90417f5d9e,fridge,not_applicable,smoke,smoke-verified,10;20;42,3,53.186684329148044,0.8083813761910171,0.7212017869840711,0.030549052359127225,11.24612321498959,0.6311736800212133,3212099.0,0.0,1329563648.0,0.0 -corrected-t1-redd,T1,corrected,PatchTST,transformer,9d8de5f72b1010780ee736900644210cd60be6f1e27f64680ed20eaa30c158b6,,825740b39bcd44b3f4bfaf146f4c0d944843b131,83fb39e57d50ac433314e880176fef187997d5b3,sha256:dd977962c2e0d72e2d923f8f5c3e92e538f67a00495e545c2f22571001872e91,NVIDIA A100-SXM4-80GB,60,299,3,3960,7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056,68326c07a5ae30495f929d97858f5b49e5cfdeaddf78334066d74b90417f5d9e,fridge,not_applicable,smoke,smoke-verified,10;20;42,3,56.397111539882644,2.9141793059289416,0.6521443272068602,0.02538089740106514,12.306998797032671,0.6492131196627466,110977.0,0.0,126057984.0,0.0 -corrected-t1-redd,T1,corrected,MSDC,specialized,9d8de5f72b1010780ee736900644210cd60be6f1e27f64680ed20eaa30c158b6,,825740b39bcd44b3f4bfaf146f4c0d944843b131,83fb39e57d50ac433314e880176fef187997d5b3,sha256:dd977962c2e0d72e2d923f8f5c3e92e538f67a00495e545c2f22571001872e91,NVIDIA A100-SXM4-80GB,60,299,3,3960,7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056,68326c07a5ae30495f929d97858f5b49e5cfdeaddf78334066d74b90417f5d9e,fridge,not_applicable,smoke,smoke-verified,10;20;42,3,58.332571615678035,0.4296383540946913,0.683834543954592,0.01482408304722735,799.208691649721,5.527490501703958,23765.0,0.0,1017177600.0,0.0 -corrected-t1-redd,T1,corrected,TCN,convolutional,9d8de5f72b1010780ee736900644210cd60be6f1e27f64680ed20eaa30c158b6,,825740b39bcd44b3f4bfaf146f4c0d944843b131,83fb39e57d50ac433314e880176fef187997d5b3,sha256:dd977962c2e0d72e2d923f8f5c3e92e538f67a00495e545c2f22571001872e91,NVIDIA A100-SXM4-80GB,60,299,3,3960,7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056,68326c07a5ae30495f929d97858f5b49e5cfdeaddf78334066d74b90417f5d9e,fridge,not_applicable,smoke,smoke-verified,10;20;42,3,58.75746796582965,7.580525322209272,0.6871346224489452,0.04896129719452161,136.11247624934185,3.1405959998419757,74151.0,0.0,242932224.0,0.0 -corrected-t1-redd,T1,corrected,ResNet,convolutional,9d8de5f72b1010780ee736900644210cd60be6f1e27f64680ed20eaa30c158b6,,825740b39bcd44b3f4bfaf146f4c0d944843b131,83fb39e57d50ac433314e880176fef187997d5b3,sha256:dd977962c2e0d72e2d923f8f5c3e92e538f67a00495e545c2f22571001872e91,NVIDIA A100-SXM4-80GB,60,299,3,3960,7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056,68326c07a5ae30495f929d97858f5b49e5cfdeaddf78334066d74b90417f5d9e,fridge,not_applicable,smoke,smoke-verified,10;20;42,3,59.01963263951128,1.5625911625144107,0.6410368172977224,0.009335530581015182,12.622285443629758,0.29376273772591066,2410409.0,0.0,1166041600.0,0.0 -corrected-t1-redd,T1,corrected,ResNetClassification,convolutional-classification,9d8de5f72b1010780ee736900644210cd60be6f1e27f64680ed20eaa30c158b6,,825740b39bcd44b3f4bfaf146f4c0d944843b131,83fb39e57d50ac433314e880176fef187997d5b3,sha256:dd977962c2e0d72e2d923f8f5c3e92e538f67a00495e545c2f22571001872e91,NVIDIA A100-SXM4-80GB,60,299,3,3960,7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056,68326c07a5ae30495f929d97858f5b49e5cfdeaddf78334066d74b90417f5d9e,fridge,not_applicable,smoke,smoke-verified,10;20;42,3,63.85270731683115,0.3908975588789727,0.0,0.0,13.406618166326856,0.5643611879407473,16387058.0,0.0,1309979136.0,0.0 -corrected-t1-redd,T1,corrected,RNNAttentionClassification,recurrent-attention-classification,9d8de5f72b1010780ee736900644210cd60be6f1e27f64680ed20eaa30c158b6,,825740b39bcd44b3f4bfaf146f4c0d944843b131,83fb39e57d50ac433314e880176fef187997d5b3,sha256:dd977962c2e0d72e2d923f8f5c3e92e538f67a00495e545c2f22571001872e91,NVIDIA A100-SXM4-80GB,60,299,3,3960,7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056,68326c07a5ae30495f929d97858f5b49e5cfdeaddf78334066d74b90417f5d9e,fridge,not_applicable,smoke,smoke-verified,10;20;42,3,64.55737705059016,0.2343551193182739,0.0,0.0,15.105366171648106,1.9433484735227533,15414725.0,0.0,12151424000.0,0.0 -corrected-t1-redd,T1,corrected,BERT,transformer,07f5562c93a226b28c0f287f72c181f6a7d322c0939fb2d5ca379e6449fc5445,,825740b39bcd44b3f4bfaf146f4c0d944843b131,83fb39e57d50ac433314e880176fef187997d5b3,sha256:dd977962c2e0d72e2d923f8f5c3e92e538f67a00495e545c2f22571001872e91,NVIDIA A100-SXM4-80GB,60,99,3,3960,fe0a640391067d0d8eef3655b630a41151a1bdf4cbff179ce47626525764d28a,81fbe9a0cd8d218ec6bbef16b85d38833c4263704915fb7564a3a69ae878ad0d,fridge,not_applicable,smoke,smoke-verified,10;20;42,3,73.74553795377541,5.186088468873656,0.4710543211931381,0.020022196781206772,11.714292020963816,0.5074728817985983,803443.0,0.0,95618048.0,0.0 -corrected-t1-redd,T1,corrected,RNNAttention,recurrent-attention,9d8de5f72b1010780ee736900644210cd60be6f1e27f64680ed20eaa30c158b6,,825740b39bcd44b3f4bfaf146f4c0d944843b131,83fb39e57d50ac433314e880176fef187997d5b3,sha256:dd977962c2e0d72e2d923f8f5c3e92e538f67a00495e545c2f22571001872e91,NVIDIA A100-SXM4-80GB,60,299,3,3960,7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056,68326c07a5ae30495f929d97858f5b49e5cfdeaddf78334066d74b90417f5d9e,fridge,not_applicable,smoke,smoke-verified,10;20;42,3,77.12848631459774,0.3383526698775547,0.4767690771960958,0.004735476087685544,13.1685075579832,1.1892074262814203,1333842.0,0.0,1612759552.0,0.0 -corrected-t1-redd,T1,corrected,Mean,statistical-baseline,44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a,,825740b39bcd44b3f4bfaf146f4c0d944843b131,83fb39e57d50ac433314e880176fef187997d5b3,sha256:dd977962c2e0d72e2d923f8f5c3e92e538f67a00495e545c2f22571001872e91,NVIDIA A100-SXM4-80GB,60,,,3960,294356c6c7b66030569075b0afb4f8937761162c74d3e189a0091ed038522ccd,f87939131f6dbb6f094f9b285d5935b5d47100608b7e3b74f99ed979aec4096a,fridge,not_applicable,smoke,smoke-verified,10;20;42,3,77.20755818231319,0.0,0.4795031055900621,0.0,9.313194934356337,1.0768063685684828,,,, -corrected-t1-redd,T1,corrected,Reformer,transformer,9d8de5f72b1010780ee736900644210cd60be6f1e27f64680ed20eaa30c158b6,,825740b39bcd44b3f4bfaf146f4c0d944843b131,83fb39e57d50ac433314e880176fef187997d5b3,sha256:dd977962c2e0d72e2d923f8f5c3e92e538f67a00495e545c2f22571001872e91,NVIDIA A100-SXM4-80GB,60,299,3,3960,7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056,68326c07a5ae30495f929d97858f5b49e5cfdeaddf78334066d74b90417f5d9e,fridge,not_applicable,smoke,smoke-verified,10;20;42,3,77.4525865206986,0.0867117316614322,0.4795031055900621,0.0,43.49176566399789,0.9955491209174842,19540993.0,0.0,26229808640.0,0.0 -corrected-t1-redd,T1,corrected,RNN,recurrent,9d8de5f72b1010780ee736900644210cd60be6f1e27f64680ed20eaa30c158b6,,825740b39bcd44b3f4bfaf146f4c0d944843b131,83fb39e57d50ac433314e880176fef187997d5b3,sha256:dd977962c2e0d72e2d923f8f5c3e92e538f67a00495e545c2f22571001872e91,NVIDIA A100-SXM4-80GB,60,299,3,3960,7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056,68326c07a5ae30495f929d97858f5b49e5cfdeaddf78334066d74b90417f5d9e,fridge,not_applicable,smoke,smoke-verified,10;20;42,3,77.5298202811438,0.5726973054994386,0.4295428301445255,0.04402182372874708,12.486517743673176,0.39894672731627295,1268049.0,0.0,3418447360.0,0.0 -corrected-t1-redd,T1,corrected,ConvLSTM,hybrid,9d8de5f72b1010780ee736900644210cd60be6f1e27f64680ed20eaa30c158b6,,825740b39bcd44b3f4bfaf146f4c0d944843b131,83fb39e57d50ac433314e880176fef187997d5b3,sha256:dd977962c2e0d72e2d923f8f5c3e92e538f67a00495e545c2f22571001872e91,NVIDIA A100-SXM4-80GB,60,299,3,3960,7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056,68326c07a5ae30495f929d97858f5b49e5cfdeaddf78334066d74b90417f5d9e,fridge,not_applicable,smoke,smoke-verified,10;20;42,3,82.82946027127097,2.914521861838068,0.2311816993335453,0.13497509930474993,13.459638093675798,1.6351937858137178,482913.0,0.0,13109322240.0,0.0 +task,family,profile,model,model_family,model_params_sha256,tuning_study_digest,model_git_sha,runner_git_sha,container_digest,hardware,sample_period,sequence_length,epochs,max_samples_per_window,protocol_overrides_sha256,comparison_protocol_sha256,ranking_protocol_sha256,rank,appliance,target_data_access,scope,status,seeds,run_count,mae_mean,mae_std,f1_mean,f1_std,elapsed_seconds_mean,elapsed_seconds_std,trainable_parameters_mean,trainable_parameters_std,peak_accelerator_memory_bytes_mean,peak_accelerator_memory_bytes_std +corrected-t1-redd,T1,corrected,NILMFormer,transformer,9d8de5f72b1010780ee736900644210cd60be6f1e27f64680ed20eaa30c158b6,,825740b39bcd44b3f4bfaf146f4c0d944843b131,83fb39e57d50ac433314e880176fef187997d5b3,sha256:dd977962c2e0d72e2d923f8f5c3e92e538f67a00495e545c2f22571001872e91,NVIDIA A100-SXM4-80GB,60,299,3,3960,7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056,68326c07a5ae30495f929d97858f5b49e5cfdeaddf78334066d74b90417f5d9e,bc8002ef4b917cda509fc789093422ad2abfb3d472bd00c7df3f846d750a14b4,1,fridge,not_applicable,smoke,smoke-verified,10;20;42,3,48.08586044774824,0.6277903001830687,0.6281731157580089,0.006296768732152522,57.085627023984365,0.9263972298357254,383283.0,0.0,4020728832.0,0.0 +corrected-t1-redd,T1,corrected,Seq2Point,convolutional,9d8de5f72b1010780ee736900644210cd60be6f1e27f64680ed20eaa30c158b6,,825740b39bcd44b3f4bfaf146f4c0d944843b131,83fb39e57d50ac433314e880176fef187997d5b3,sha256:dd977962c2e0d72e2d923f8f5c3e92e538f67a00495e545c2f22571001872e91,NVIDIA A100-SXM4-80GB,60,299,3,3960,7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056,68326c07a5ae30495f929d97858f5b49e5cfdeaddf78334066d74b90417f5d9e,bc8002ef4b917cda509fc789093422ad2abfb3d472bd00c7df3f846d750a14b4,2,fridge,not_applicable,smoke,smoke-verified,10;20;42,3,49.13913551508602,1.368821477825844,0.738922168844469,0.02550131120698661,11.937222305607671,0.9718365327389646,13863449.0,0.0,724345856.0,0.0 +corrected-t1-redd,T1,corrected,DAE,autoencoder,07f5562c93a226b28c0f287f72c181f6a7d322c0939fb2d5ca379e6449fc5445,,825740b39bcd44b3f4bfaf146f4c0d944843b131,83fb39e57d50ac433314e880176fef187997d5b3,sha256:dd977962c2e0d72e2d923f8f5c3e92e538f67a00495e545c2f22571001872e91,NVIDIA A100-SXM4-80GB,60,99,3,3960,fe0a640391067d0d8eef3655b630a41151a1bdf4cbff179ce47626525764d28a,81fbe9a0cd8d218ec6bbef16b85d38833c4263704915fb7564a3a69ae878ad0d,bc8002ef4b917cda509fc789093422ad2abfb3d472bd00c7df3f846d750a14b4,3,fridge,not_applicable,smoke,smoke-verified,10;20;42,3,49.81314141919794,1.2750812946872356,0.7096876676310145,0.011508582889399363,11.623419809659632,1.069115608188479,831801.0,0.0,83911168.0,0.0 +corrected-t1-redd,T1,corrected,WindowGRU,recurrent,9d8de5f72b1010780ee736900644210cd60be6f1e27f64680ed20eaa30c158b6,,825740b39bcd44b3f4bfaf146f4c0d944843b131,83fb39e57d50ac433314e880176fef187997d5b3,sha256:dd977962c2e0d72e2d923f8f5c3e92e538f67a00495e545c2f22571001872e91,NVIDIA A100-SXM4-80GB,60,299,3,3960,7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056,68326c07a5ae30495f929d97858f5b49e5cfdeaddf78334066d74b90417f5d9e,bc8002ef4b917cda509fc789093422ad2abfb3d472bd00c7df3f846d750a14b4,4,fridge,not_applicable,smoke,smoke-verified,10;20;42,3,50.47948637432652,1.3556019544833675,0.6971819197814662,0.007023031532984379,11.894063677988015,0.13453522512836433,427345.0,0.0,2746802688.0,0.0 +corrected-t1-redd,T1,corrected,Seq2Seq,recurrent,9d8de5f72b1010780ee736900644210cd60be6f1e27f64680ed20eaa30c158b6,,825740b39bcd44b3f4bfaf146f4c0d944843b131,83fb39e57d50ac433314e880176fef187997d5b3,sha256:dd977962c2e0d72e2d923f8f5c3e92e538f67a00495e545c2f22571001872e91,NVIDIA A100-SXM4-80GB,60,299,3,3960,7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056,68326c07a5ae30495f929d97858f5b49e5cfdeaddf78334066d74b90417f5d9e,bc8002ef4b917cda509fc789093422ad2abfb3d472bd00c7df3f846d750a14b4,5,fridge,not_applicable,smoke,smoke-verified,10;20;42,3,53.186684329148044,0.8083813761910171,0.7212017869840711,0.030549052359127225,11.24612321498959,0.6311736800212133,3212099.0,0.0,1329563648.0,0.0 +corrected-t1-redd,T1,corrected,PatchTST,transformer,9d8de5f72b1010780ee736900644210cd60be6f1e27f64680ed20eaa30c158b6,,825740b39bcd44b3f4bfaf146f4c0d944843b131,83fb39e57d50ac433314e880176fef187997d5b3,sha256:dd977962c2e0d72e2d923f8f5c3e92e538f67a00495e545c2f22571001872e91,NVIDIA A100-SXM4-80GB,60,299,3,3960,7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056,68326c07a5ae30495f929d97858f5b49e5cfdeaddf78334066d74b90417f5d9e,bc8002ef4b917cda509fc789093422ad2abfb3d472bd00c7df3f846d750a14b4,6,fridge,not_applicable,smoke,smoke-verified,10;20;42,3,56.397111539882644,2.9141793059289416,0.6521443272068602,0.02538089740106514,12.306998797032671,0.6492131196627466,110977.0,0.0,126057984.0,0.0 +corrected-t1-redd,T1,corrected,MSDC,specialized,9d8de5f72b1010780ee736900644210cd60be6f1e27f64680ed20eaa30c158b6,,825740b39bcd44b3f4bfaf146f4c0d944843b131,83fb39e57d50ac433314e880176fef187997d5b3,sha256:dd977962c2e0d72e2d923f8f5c3e92e538f67a00495e545c2f22571001872e91,NVIDIA A100-SXM4-80GB,60,299,3,3960,7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056,68326c07a5ae30495f929d97858f5b49e5cfdeaddf78334066d74b90417f5d9e,bc8002ef4b917cda509fc789093422ad2abfb3d472bd00c7df3f846d750a14b4,7,fridge,not_applicable,smoke,smoke-verified,10;20;42,3,58.332571615678035,0.4296383540946913,0.683834543954592,0.01482408304722735,799.208691649721,5.527490501703958,23765.0,0.0,1017177600.0,0.0 +corrected-t1-redd,T1,corrected,TCN,convolutional,9d8de5f72b1010780ee736900644210cd60be6f1e27f64680ed20eaa30c158b6,,825740b39bcd44b3f4bfaf146f4c0d944843b131,83fb39e57d50ac433314e880176fef187997d5b3,sha256:dd977962c2e0d72e2d923f8f5c3e92e538f67a00495e545c2f22571001872e91,NVIDIA A100-SXM4-80GB,60,299,3,3960,7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056,68326c07a5ae30495f929d97858f5b49e5cfdeaddf78334066d74b90417f5d9e,bc8002ef4b917cda509fc789093422ad2abfb3d472bd00c7df3f846d750a14b4,8,fridge,not_applicable,smoke,smoke-verified,10;20;42,3,58.75746796582965,7.580525322209272,0.6871346224489452,0.04896129719452161,136.11247624934185,3.1405959998419757,74151.0,0.0,242932224.0,0.0 +corrected-t1-redd,T1,corrected,ResNet,convolutional,9d8de5f72b1010780ee736900644210cd60be6f1e27f64680ed20eaa30c158b6,,825740b39bcd44b3f4bfaf146f4c0d944843b131,83fb39e57d50ac433314e880176fef187997d5b3,sha256:dd977962c2e0d72e2d923f8f5c3e92e538f67a00495e545c2f22571001872e91,NVIDIA A100-SXM4-80GB,60,299,3,3960,7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056,68326c07a5ae30495f929d97858f5b49e5cfdeaddf78334066d74b90417f5d9e,bc8002ef4b917cda509fc789093422ad2abfb3d472bd00c7df3f846d750a14b4,9,fridge,not_applicable,smoke,smoke-verified,10;20;42,3,59.01963263951128,1.5625911625144107,0.6410368172977224,0.009335530581015182,12.622285443629758,0.29376273772591066,2410409.0,0.0,1166041600.0,0.0 +corrected-t1-redd,T1,corrected,ResNetClassification,convolutional-classification,9d8de5f72b1010780ee736900644210cd60be6f1e27f64680ed20eaa30c158b6,,825740b39bcd44b3f4bfaf146f4c0d944843b131,83fb39e57d50ac433314e880176fef187997d5b3,sha256:dd977962c2e0d72e2d923f8f5c3e92e538f67a00495e545c2f22571001872e91,NVIDIA A100-SXM4-80GB,60,299,3,3960,7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056,68326c07a5ae30495f929d97858f5b49e5cfdeaddf78334066d74b90417f5d9e,bc8002ef4b917cda509fc789093422ad2abfb3d472bd00c7df3f846d750a14b4,10,fridge,not_applicable,smoke,smoke-verified,10;20;42,3,63.85270731683115,0.3908975588789727,0.0,0.0,13.406618166326856,0.5643611879407473,16387058.0,0.0,1309979136.0,0.0 +corrected-t1-redd,T1,corrected,RNNAttentionClassification,recurrent-attention-classification,9d8de5f72b1010780ee736900644210cd60be6f1e27f64680ed20eaa30c158b6,,825740b39bcd44b3f4bfaf146f4c0d944843b131,83fb39e57d50ac433314e880176fef187997d5b3,sha256:dd977962c2e0d72e2d923f8f5c3e92e538f67a00495e545c2f22571001872e91,NVIDIA A100-SXM4-80GB,60,299,3,3960,7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056,68326c07a5ae30495f929d97858f5b49e5cfdeaddf78334066d74b90417f5d9e,bc8002ef4b917cda509fc789093422ad2abfb3d472bd00c7df3f846d750a14b4,11,fridge,not_applicable,smoke,smoke-verified,10;20;42,3,64.55737705059016,0.2343551193182739,0.0,0.0,15.105366171648106,1.9433484735227533,15414725.0,0.0,12151424000.0,0.0 +corrected-t1-redd,T1,corrected,BERT,transformer,07f5562c93a226b28c0f287f72c181f6a7d322c0939fb2d5ca379e6449fc5445,,825740b39bcd44b3f4bfaf146f4c0d944843b131,83fb39e57d50ac433314e880176fef187997d5b3,sha256:dd977962c2e0d72e2d923f8f5c3e92e538f67a00495e545c2f22571001872e91,NVIDIA A100-SXM4-80GB,60,99,3,3960,fe0a640391067d0d8eef3655b630a41151a1bdf4cbff179ce47626525764d28a,81fbe9a0cd8d218ec6bbef16b85d38833c4263704915fb7564a3a69ae878ad0d,bc8002ef4b917cda509fc789093422ad2abfb3d472bd00c7df3f846d750a14b4,12,fridge,not_applicable,smoke,smoke-verified,10;20;42,3,73.74553795377541,5.186088468873656,0.4710543211931381,0.020022196781206772,11.714292020963816,0.5074728817985983,803443.0,0.0,95618048.0,0.0 +corrected-t1-redd,T1,corrected,RNNAttention,recurrent-attention,9d8de5f72b1010780ee736900644210cd60be6f1e27f64680ed20eaa30c158b6,,825740b39bcd44b3f4bfaf146f4c0d944843b131,83fb39e57d50ac433314e880176fef187997d5b3,sha256:dd977962c2e0d72e2d923f8f5c3e92e538f67a00495e545c2f22571001872e91,NVIDIA A100-SXM4-80GB,60,299,3,3960,7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056,68326c07a5ae30495f929d97858f5b49e5cfdeaddf78334066d74b90417f5d9e,bc8002ef4b917cda509fc789093422ad2abfb3d472bd00c7df3f846d750a14b4,13,fridge,not_applicable,smoke,smoke-verified,10;20;42,3,77.12848631459774,0.3383526698775547,0.4767690771960958,0.004735476087685544,13.1685075579832,1.1892074262814203,1333842.0,0.0,1612759552.0,0.0 +corrected-t1-redd,T1,corrected,Mean,statistical-baseline,44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a,,825740b39bcd44b3f4bfaf146f4c0d944843b131,83fb39e57d50ac433314e880176fef187997d5b3,sha256:dd977962c2e0d72e2d923f8f5c3e92e538f67a00495e545c2f22571001872e91,NVIDIA A100-SXM4-80GB,60,,,3960,294356c6c7b66030569075b0afb4f8937761162c74d3e189a0091ed038522ccd,f87939131f6dbb6f094f9b285d5935b5d47100608b7e3b74f99ed979aec4096a,bc8002ef4b917cda509fc789093422ad2abfb3d472bd00c7df3f846d750a14b4,14,fridge,not_applicable,smoke,smoke-verified,10;20;42,3,77.20755818231319,0.0,0.4795031055900621,0.0,9.313194934356337,1.0768063685684828,,,, +corrected-t1-redd,T1,corrected,Reformer,transformer,9d8de5f72b1010780ee736900644210cd60be6f1e27f64680ed20eaa30c158b6,,825740b39bcd44b3f4bfaf146f4c0d944843b131,83fb39e57d50ac433314e880176fef187997d5b3,sha256:dd977962c2e0d72e2d923f8f5c3e92e538f67a00495e545c2f22571001872e91,NVIDIA A100-SXM4-80GB,60,299,3,3960,7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056,68326c07a5ae30495f929d97858f5b49e5cfdeaddf78334066d74b90417f5d9e,bc8002ef4b917cda509fc789093422ad2abfb3d472bd00c7df3f846d750a14b4,15,fridge,not_applicable,smoke,smoke-verified,10;20;42,3,77.4525865206986,0.0867117316614322,0.4795031055900621,0.0,43.49176566399789,0.9955491209174842,19540993.0,0.0,26229808640.0,0.0 +corrected-t1-redd,T1,corrected,RNN,recurrent,9d8de5f72b1010780ee736900644210cd60be6f1e27f64680ed20eaa30c158b6,,825740b39bcd44b3f4bfaf146f4c0d944843b131,83fb39e57d50ac433314e880176fef187997d5b3,sha256:dd977962c2e0d72e2d923f8f5c3e92e538f67a00495e545c2f22571001872e91,NVIDIA A100-SXM4-80GB,60,299,3,3960,7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056,68326c07a5ae30495f929d97858f5b49e5cfdeaddf78334066d74b90417f5d9e,bc8002ef4b917cda509fc789093422ad2abfb3d472bd00c7df3f846d750a14b4,16,fridge,not_applicable,smoke,smoke-verified,10;20;42,3,77.5298202811438,0.5726973054994386,0.4295428301445255,0.04402182372874708,12.486517743673176,0.39894672731627295,1268049.0,0.0,3418447360.0,0.0 +corrected-t1-redd,T1,corrected,ConvLSTM,hybrid,9d8de5f72b1010780ee736900644210cd60be6f1e27f64680ed20eaa30c158b6,,825740b39bcd44b3f4bfaf146f4c0d944843b131,83fb39e57d50ac433314e880176fef187997d5b3,sha256:dd977962c2e0d72e2d923f8f5c3e92e538f67a00495e545c2f22571001872e91,NVIDIA A100-SXM4-80GB,60,299,3,3960,7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056,68326c07a5ae30495f929d97858f5b49e5cfdeaddf78334066d74b90417f5d9e,bc8002ef4b917cda509fc789093422ad2abfb3d472bd00c7df3f846d750a14b4,17,fridge,not_applicable,smoke,smoke-verified,10;20;42,3,82.82946027127097,2.914521861838068,0.2311816993335453,0.13497509930474993,13.459638093675798,1.6351937858137178,482913.0,0.0,13109322240.0,0.0 diff --git a/leaderboard.json b/leaderboard.json index 7116453..8c2fa57 100644 --- a/leaderboard.json +++ b/leaderboard.json @@ -1,7 +1,7 @@ { "artifacts": { "csv_name": "leaderboard.csv", - "csv_sha256": "130a6394bc1f041491bb33a840d1d37a2350aadaf2673f8e59b0f2c96e24af49" + "csv_sha256": "7693db2ccd0c03581fd5c1d9ae53b3f3bb083c56a4fb331fca939690eeb766dc" }, "entries": [ { @@ -69,6 +69,19 @@ }, "protocol_overrides_sha256": "7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056", "provenance_verified": true, + "rank": 1, + "ranking_protocol": { + "appliance": "fridge", + "max_samples_per_window": 3960, + "profile": "corrected", + "sample_period": 60, + "schema": "nilmbench.ranking-protocol.v1", + "scope": "smoke", + "target_data_access": "not_applicable", + "task_config_sha256": "95088d9a943c124c25a2cfada707f34088c178ee85b78da9052edbd2cc69fea4", + "task_id": "corrected-t1-redd" + }, + "ranking_protocol_sha256": "bc8002ef4b917cda509fc789093422ad2abfb3d472bd00c7df3f846d750a14b4", "result_file_sha256": [ "73030fec03b2384c74de17c001c09e40f97cc949e4ed0982bfd95af1a721635e", "863e4dd1c3ef33e0ed52697a63a5ad6db742bba14122f660a471a58d23ad143c", @@ -163,6 +176,19 @@ }, "protocol_overrides_sha256": "7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056", "provenance_verified": true, + "rank": 2, + "ranking_protocol": { + "appliance": "fridge", + "max_samples_per_window": 3960, + "profile": "corrected", + "sample_period": 60, + "schema": "nilmbench.ranking-protocol.v1", + "scope": "smoke", + "target_data_access": "not_applicable", + "task_config_sha256": "95088d9a943c124c25a2cfada707f34088c178ee85b78da9052edbd2cc69fea4", + "task_id": "corrected-t1-redd" + }, + "ranking_protocol_sha256": "bc8002ef4b917cda509fc789093422ad2abfb3d472bd00c7df3f846d750a14b4", "result_file_sha256": [ "2f2d7481260deae279aa0e1d7b376d50a2362bae2e0a11243a757a27f7e13064", "5f58b3b479d236284b5b5666e91dfb969fea98cadc34b2e2b394e2ff9b1fd2c5", @@ -257,6 +283,19 @@ }, "protocol_overrides_sha256": "fe0a640391067d0d8eef3655b630a41151a1bdf4cbff179ce47626525764d28a", "provenance_verified": true, + "rank": 3, + "ranking_protocol": { + "appliance": "fridge", + "max_samples_per_window": 3960, + "profile": "corrected", + "sample_period": 60, + "schema": "nilmbench.ranking-protocol.v1", + "scope": "smoke", + "target_data_access": "not_applicable", + "task_config_sha256": "95088d9a943c124c25a2cfada707f34088c178ee85b78da9052edbd2cc69fea4", + "task_id": "corrected-t1-redd" + }, + "ranking_protocol_sha256": "bc8002ef4b917cda509fc789093422ad2abfb3d472bd00c7df3f846d750a14b4", "result_file_sha256": [ "9079482cff7ca1fb426dc518c6959bd036f49ea90de9caf9f1bcd62896ebdbd1", "b41f75deb3ffa243475b97859ce536b0570296aa81061a4a74e66504d3a2753e", @@ -351,6 +390,19 @@ }, "protocol_overrides_sha256": "7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056", "provenance_verified": true, + "rank": 4, + "ranking_protocol": { + "appliance": "fridge", + "max_samples_per_window": 3960, + "profile": "corrected", + "sample_period": 60, + "schema": "nilmbench.ranking-protocol.v1", + "scope": "smoke", + "target_data_access": "not_applicable", + "task_config_sha256": "95088d9a943c124c25a2cfada707f34088c178ee85b78da9052edbd2cc69fea4", + "task_id": "corrected-t1-redd" + }, + "ranking_protocol_sha256": "bc8002ef4b917cda509fc789093422ad2abfb3d472bd00c7df3f846d750a14b4", "result_file_sha256": [ "4cb9bde15cab2fa76b1b330596bfd5118a0e309c5d0e29a6b4e1216486df244e", "a9e566210b505ab752a12ae9ba40e9cd0595226b70759d938b7d698619288a05", @@ -445,6 +497,19 @@ }, "protocol_overrides_sha256": "7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056", "provenance_verified": true, + "rank": 5, + "ranking_protocol": { + "appliance": "fridge", + "max_samples_per_window": 3960, + "profile": "corrected", + "sample_period": 60, + "schema": "nilmbench.ranking-protocol.v1", + "scope": "smoke", + "target_data_access": "not_applicable", + "task_config_sha256": "95088d9a943c124c25a2cfada707f34088c178ee85b78da9052edbd2cc69fea4", + "task_id": "corrected-t1-redd" + }, + "ranking_protocol_sha256": "bc8002ef4b917cda509fc789093422ad2abfb3d472bd00c7df3f846d750a14b4", "result_file_sha256": [ "0d3d123799624431e17d791735dd2029d5086ca9317afa63d9e6d992be1122c1", "c580e6570b7f0251d57fff01c097279acab6327aadc4d6ead0cd68ce3eeed2ba", @@ -539,6 +604,19 @@ }, "protocol_overrides_sha256": "7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056", "provenance_verified": true, + "rank": 6, + "ranking_protocol": { + "appliance": "fridge", + "max_samples_per_window": 3960, + "profile": "corrected", + "sample_period": 60, + "schema": "nilmbench.ranking-protocol.v1", + "scope": "smoke", + "target_data_access": "not_applicable", + "task_config_sha256": "95088d9a943c124c25a2cfada707f34088c178ee85b78da9052edbd2cc69fea4", + "task_id": "corrected-t1-redd" + }, + "ranking_protocol_sha256": "bc8002ef4b917cda509fc789093422ad2abfb3d472bd00c7df3f846d750a14b4", "result_file_sha256": [ "645467b57a9f3901c9d7dfeb4678f38e8271cca4ac949bea05f34af905ba0db7", "a7025964e7d8abe80996911c793d4f65f7381007863f456f8e6da48b62187aef", @@ -633,6 +711,19 @@ }, "protocol_overrides_sha256": "7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056", "provenance_verified": true, + "rank": 7, + "ranking_protocol": { + "appliance": "fridge", + "max_samples_per_window": 3960, + "profile": "corrected", + "sample_period": 60, + "schema": "nilmbench.ranking-protocol.v1", + "scope": "smoke", + "target_data_access": "not_applicable", + "task_config_sha256": "95088d9a943c124c25a2cfada707f34088c178ee85b78da9052edbd2cc69fea4", + "task_id": "corrected-t1-redd" + }, + "ranking_protocol_sha256": "bc8002ef4b917cda509fc789093422ad2abfb3d472bd00c7df3f846d750a14b4", "result_file_sha256": [ "155f158fb13e12d1f701307fd81b312e61884faadf4bdb5d2d6c0f8242f66012", "27f781248b31b2b7e5cc89723852de739303be951c9a36e606aea6fdae254e81", @@ -727,6 +818,19 @@ }, "protocol_overrides_sha256": "7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056", "provenance_verified": true, + "rank": 8, + "ranking_protocol": { + "appliance": "fridge", + "max_samples_per_window": 3960, + "profile": "corrected", + "sample_period": 60, + "schema": "nilmbench.ranking-protocol.v1", + "scope": "smoke", + "target_data_access": "not_applicable", + "task_config_sha256": "95088d9a943c124c25a2cfada707f34088c178ee85b78da9052edbd2cc69fea4", + "task_id": "corrected-t1-redd" + }, + "ranking_protocol_sha256": "bc8002ef4b917cda509fc789093422ad2abfb3d472bd00c7df3f846d750a14b4", "result_file_sha256": [ "0ac7552de10c0b19e54d36673a6982da479a1831fc29d0deba756e0dfdcbf27f", "38cec1110bcb4e53ca0f49388b37b13e8c3809868ba7bd1086d5576f29a67895", @@ -821,6 +925,19 @@ }, "protocol_overrides_sha256": "7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056", "provenance_verified": true, + "rank": 9, + "ranking_protocol": { + "appliance": "fridge", + "max_samples_per_window": 3960, + "profile": "corrected", + "sample_period": 60, + "schema": "nilmbench.ranking-protocol.v1", + "scope": "smoke", + "target_data_access": "not_applicable", + "task_config_sha256": "95088d9a943c124c25a2cfada707f34088c178ee85b78da9052edbd2cc69fea4", + "task_id": "corrected-t1-redd" + }, + "ranking_protocol_sha256": "bc8002ef4b917cda509fc789093422ad2abfb3d472bd00c7df3f846d750a14b4", "result_file_sha256": [ "6290dd5922d63077b44f76ebe87702b88cd20457ea24c5303b847732dd84920a", "e52cb6ef8f2b31b47ffbf30ea66f94896e7e5bd510906f2f199e17eb03f911a7", @@ -915,6 +1032,19 @@ }, "protocol_overrides_sha256": "7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056", "provenance_verified": true, + "rank": 10, + "ranking_protocol": { + "appliance": "fridge", + "max_samples_per_window": 3960, + "profile": "corrected", + "sample_period": 60, + "schema": "nilmbench.ranking-protocol.v1", + "scope": "smoke", + "target_data_access": "not_applicable", + "task_config_sha256": "95088d9a943c124c25a2cfada707f34088c178ee85b78da9052edbd2cc69fea4", + "task_id": "corrected-t1-redd" + }, + "ranking_protocol_sha256": "bc8002ef4b917cda509fc789093422ad2abfb3d472bd00c7df3f846d750a14b4", "result_file_sha256": [ "8065e7bd11a837aac7140b77a4e011441519304aeb1c8f189fac40103c593326", "d37b82aa6f7aa6ea1be49b37de896a62b685a15645a6f59728e629a420cbddf5", @@ -1009,6 +1139,19 @@ }, "protocol_overrides_sha256": "7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056", "provenance_verified": true, + "rank": 11, + "ranking_protocol": { + "appliance": "fridge", + "max_samples_per_window": 3960, + "profile": "corrected", + "sample_period": 60, + "schema": "nilmbench.ranking-protocol.v1", + "scope": "smoke", + "target_data_access": "not_applicable", + "task_config_sha256": "95088d9a943c124c25a2cfada707f34088c178ee85b78da9052edbd2cc69fea4", + "task_id": "corrected-t1-redd" + }, + "ranking_protocol_sha256": "bc8002ef4b917cda509fc789093422ad2abfb3d472bd00c7df3f846d750a14b4", "result_file_sha256": [ "2aa65755cf4eb64e175ef7ef77de0cd1d017efd025bb0a87f86e5cbf1bfedd75", "688b313658b15abdbb016b8d4e369985076e1b2b56a437a1ccf7e729bc64c045", @@ -1103,6 +1246,19 @@ }, "protocol_overrides_sha256": "fe0a640391067d0d8eef3655b630a41151a1bdf4cbff179ce47626525764d28a", "provenance_verified": true, + "rank": 12, + "ranking_protocol": { + "appliance": "fridge", + "max_samples_per_window": 3960, + "profile": "corrected", + "sample_period": 60, + "schema": "nilmbench.ranking-protocol.v1", + "scope": "smoke", + "target_data_access": "not_applicable", + "task_config_sha256": "95088d9a943c124c25a2cfada707f34088c178ee85b78da9052edbd2cc69fea4", + "task_id": "corrected-t1-redd" + }, + "ranking_protocol_sha256": "bc8002ef4b917cda509fc789093422ad2abfb3d472bd00c7df3f846d750a14b4", "result_file_sha256": [ "a7aca7a81942e4133c386bfe36cd795388857071c6dbe390e3591a92b893593f", "c70cdd2d44c65d4c177d55701f3307baac6e466e47d5bf5de7734e452a696bd1", @@ -1197,6 +1353,19 @@ }, "protocol_overrides_sha256": "7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056", "provenance_verified": true, + "rank": 13, + "ranking_protocol": { + "appliance": "fridge", + "max_samples_per_window": 3960, + "profile": "corrected", + "sample_period": 60, + "schema": "nilmbench.ranking-protocol.v1", + "scope": "smoke", + "target_data_access": "not_applicable", + "task_config_sha256": "95088d9a943c124c25a2cfada707f34088c178ee85b78da9052edbd2cc69fea4", + "task_id": "corrected-t1-redd" + }, + "ranking_protocol_sha256": "bc8002ef4b917cda509fc789093422ad2abfb3d472bd00c7df3f846d750a14b4", "result_file_sha256": [ "421844be6c76e2d9dbbd08d6ba1ccec73f1703680aa4ef234aedd5dc8141adfc", "65a84deccc30f630796307b1983319f0a3ad679db392b37c3dad52ec3d6701cc", @@ -1285,6 +1454,19 @@ }, "protocol_overrides_sha256": "294356c6c7b66030569075b0afb4f8937761162c74d3e189a0091ed038522ccd", "provenance_verified": true, + "rank": 14, + "ranking_protocol": { + "appliance": "fridge", + "max_samples_per_window": 3960, + "profile": "corrected", + "sample_period": 60, + "schema": "nilmbench.ranking-protocol.v1", + "scope": "smoke", + "target_data_access": "not_applicable", + "task_config_sha256": "95088d9a943c124c25a2cfada707f34088c178ee85b78da9052edbd2cc69fea4", + "task_id": "corrected-t1-redd" + }, + "ranking_protocol_sha256": "bc8002ef4b917cda509fc789093422ad2abfb3d472bd00c7df3f846d750a14b4", "result_file_sha256": [ "c46e149a787bc9c65de81676f8fddab18a6ef248d9d92cfdc4288d66bad06c63", "ca9081995565070588fa8ae895271f52bf62e1b6a8fc998b390e466decb957b9", @@ -1379,6 +1561,19 @@ }, "protocol_overrides_sha256": "7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056", "provenance_verified": true, + "rank": 15, + "ranking_protocol": { + "appliance": "fridge", + "max_samples_per_window": 3960, + "profile": "corrected", + "sample_period": 60, + "schema": "nilmbench.ranking-protocol.v1", + "scope": "smoke", + "target_data_access": "not_applicable", + "task_config_sha256": "95088d9a943c124c25a2cfada707f34088c178ee85b78da9052edbd2cc69fea4", + "task_id": "corrected-t1-redd" + }, + "ranking_protocol_sha256": "bc8002ef4b917cda509fc789093422ad2abfb3d472bd00c7df3f846d750a14b4", "result_file_sha256": [ "3b16687835429c14148d073fc3c6f61478af88d973b9a5812dc13cd0ed71fc22", "403f278660ea83b33e9bb0eb7cbac8d05ae9c07fcf330acb43a3aa9562b459a7", @@ -1473,6 +1668,19 @@ }, "protocol_overrides_sha256": "7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056", "provenance_verified": true, + "rank": 16, + "ranking_protocol": { + "appliance": "fridge", + "max_samples_per_window": 3960, + "profile": "corrected", + "sample_period": 60, + "schema": "nilmbench.ranking-protocol.v1", + "scope": "smoke", + "target_data_access": "not_applicable", + "task_config_sha256": "95088d9a943c124c25a2cfada707f34088c178ee85b78da9052edbd2cc69fea4", + "task_id": "corrected-t1-redd" + }, + "ranking_protocol_sha256": "bc8002ef4b917cda509fc789093422ad2abfb3d472bd00c7df3f846d750a14b4", "result_file_sha256": [ "2c3e4e96326c683130cac8092c53b9abd5be2b9784113a37927b84be350dd9e1", "45379dda3b95bf308daf7a2fadfb1facd908c8aaec0883e61cadf4b963de2a5a", @@ -1567,6 +1775,19 @@ }, "protocol_overrides_sha256": "7f654b6d261e1d9a6e733400a6ee383eacf61da0a0f5e45ce5aa7bc88f385056", "provenance_verified": true, + "rank": 17, + "ranking_protocol": { + "appliance": "fridge", + "max_samples_per_window": 3960, + "profile": "corrected", + "sample_period": 60, + "schema": "nilmbench.ranking-protocol.v1", + "scope": "smoke", + "target_data_access": "not_applicable", + "task_config_sha256": "95088d9a943c124c25a2cfada707f34088c178ee85b78da9052edbd2cc69fea4", + "task_id": "corrected-t1-redd" + }, + "ranking_protocol_sha256": "bc8002ef4b917cda509fc789093422ad2abfb3d472bd00c7df3f846d750a14b4", "result_file_sha256": [ "442d2b34f4bdcbd69f62e7442783ffe41192e93b3d57bdd2965cf218985aef36", "9714888b16cbdf016a4d39b1783b2e0e58fe7a480579564e9135a2eeae839d13", diff --git a/src/nilmbench/leaderboard.py b/src/nilmbench/leaderboard.py index 0a29e51..dd3dd93 100644 --- a/src/nilmbench/leaderboard.py +++ b/src/nilmbench/leaderboard.py @@ -69,7 +69,7 @@ def _expected_window_samples(window: Any, sample_period: int, limit: int | None) def _comparison_protocol(result: dict[str, Any]) -> dict[str, Any]: - """Return only evaluation controls under which models may be ranked together.""" + """Return the full training, runtime, and evaluation comparison record.""" overrides = result["protocol_overrides"] task = result["task"] runtime = result["runtime"] @@ -107,6 +107,27 @@ def _comparison_protocol(result: dict[str, Any]) -> dict[str, Any]: } +def _ranking_protocol(entry: dict[str, Any]) -> dict[str, Any]: + """Return evaluation controls that define one public ranking cohort. + + Model choices such as sequence length and epoch count, and provenance such as + the container revision, remain in the full comparison protocol. They must not + create a fresh public rank: otherwise every architecture or later image build + can become an incomparable group of one. + """ + return { + "schema": "nilmbench.ranking-protocol.v1", + "task_id": entry["task"], + "task_config_sha256": entry["task_config_sha256"], + "profile": entry["profile"], + "sample_period": entry["sample_period"], + "appliance": entry["appliance"], + "max_samples_per_window": entry["max_samples_per_window"], + "scope": entry["scope"], + "target_data_access": entry["target_data_access"], + } + + def _validate_study_contract( result: dict[str, Any], path: Path, config: BenchmarkConfig, task: Any ) -> None: @@ -1073,6 +1094,11 @@ def build_leaderboard( "result_file_sha256": sorted(row["result_file_sha256"] for row in rows), } ) + for entry in entries: + ranking_protocol = _ranking_protocol(entry) + entry["ranking_protocol"] = ranking_protocol + entry["ranking_protocol_sha256"] = canonical_digest(ranking_protocol) + entries.sort( key=lambda item: ( item["task"], @@ -1083,6 +1109,14 @@ def build_leaderboard( item["model"], ) ) + ranks: dict[str, int] = defaultdict(int) + for entry in entries: + if entry["status"] not in {"full-verified", "smoke-verified"}: + entry["rank"] = None + continue + cohort = entry["ranking_protocol_sha256"] + ranks[cohort] += 1 + entry["rank"] = ranks[cohort] return { "schema_version": "1.0", "required_seeds": list(required_seeds), @@ -1118,6 +1152,8 @@ def write_leaderboard( "max_samples_per_window", "protocol_overrides_sha256", "comparison_protocol_sha256", + "ranking_protocol_sha256", + "rank", "appliance", "target_data_access", "scope", diff --git a/tests/test_leaderboard.py b/tests/test_leaderboard.py index eb77c0a..87f03fa 100644 --- a/tests/test_leaderboard.py +++ b/tests/test_leaderboard.py @@ -480,6 +480,10 @@ def test_different_container_digests_are_never_aggregated(tmp_path): len({entry["comparison_protocol_sha256"] for entry in leaderboard["entries"]}) == 2 ) + assert ( + len({entry["ranking_protocol_sha256"] for entry in leaderboard["entries"]}) + == 1 + ) def test_smoke_protocol_overrides_are_never_aggregated(tmp_path): @@ -506,6 +510,37 @@ def test_smoke_protocol_overrides_are_never_aggregated(tmp_path): assert len({entry["comparison_protocol_sha256"] for entry in entries}) == 2 +def test_model_sequence_length_does_not_reset_public_rank(tmp_path): + for seed in (10, 20, 42): + _write_result( + tmp_path / "short-sequence", + seed, + scope="smoke", + sequence_length=99, + max_samples=1024, + mae=40.0, + ) + _write_result( + tmp_path / "long-sequence", + seed, + scope="smoke", + sequence_length=299, + max_samples=1024, + mae=50.0, + ) + + entries = _build(tmp_path)["entries"] + + assert [entry["sequence_length"] for entry in entries] == [99, 299] + assert len({entry["comparison_protocol_sha256"] for entry in entries}) == 2 + assert len({entry["ranking_protocol_sha256"] for entry in entries}) == 1 + assert [entry["rank"] for entry in entries] == [1, 2] + ranking_protocol = entries[0]["ranking_protocol"] + assert ranking_protocol["max_samples_per_window"] == 1024 + assert "effective_sequence_length" not in ranking_protocol + assert "runtime" not in ranking_protocol + + def test_json_and_csv_artifacts_are_deterministic(tmp_path): for seed in (10, 20, 42): _write_result(tmp_path / "results", seed) @@ -522,6 +557,8 @@ def test_json_and_csv_artifacts_are_deterministic(tmp_path): assert csv_path.read_bytes() == first_csv assert b"full-verified" in first_json assert b"full-verified" in first_csv + assert b"ranking_protocol_sha256" in first_csv + assert b",rank," in first_csv payload = json.loads(first_json) assert payload["artifacts"]["csv_name"] == "leaderboard.csv" assert payload["artifacts"]["csv_sha256"] == hashlib.sha256(first_csv).hexdigest() diff --git a/tests/test_site.py b/tests/test_site.py index 05d91a0..cfd7d1e 100644 --- a/tests/test_site.py +++ b/tests/test_site.py @@ -13,15 +13,12 @@ def test_site_loads_generated_leaderboard_and_discloses_smoke_protocol(): assert "entry.trainable_parameters_mean" in source assert "entry.elapsed_seconds_mean" in source assert "entry.peak_accelerator_memory_bytes_mean" in source - assert "comparisonKey(entry)" in source - comparison_key = source.split("function comparisonKey(entry) {", 1)[1].split( - "}", 1 - )[0] - assert "entry.comparison_protocol_sha256" in comparison_key - assert "entry.tuning_study_digest" not in comparison_key - assert "entry.model_params_sha256" not in comparison_key + assert "ranking_protocol_sha256" in source + assert "Number.isInteger(entry.rank)" in source + assert "comparisonKey(entry)" not in source + assert "const ranks = new Map()" not in source assert "entry.max_samples_per_window ?? 'full'" not in source - assert "Group rank" in source + assert "" in source assert "entries.slice(0, 25)" not in source assert ( "nilmbench[benchmark] @ git+https://github.com/sustainability-lab/nilmbench.git"
Group rankModelTask / applianceMAE↓ (W)F1↑ParamsTime↓Peak VRAM↓Status
RankModelTask / applianceMAE↓ (W)F1↑ParamsTime↓Peak VRAM↓Status
Loading…
${rank ?? '—'}${escHTML(entry.model)}${escHTML(task)}Rank